From 64bcbe2e83e572f583e4c1ed90be18a056ee86a5 Mon Sep 17 00:00:00 2001 From: yuema137 <3124558229@qq.com> Date: Fri, 19 Apr 2024 18:30:28 -0500 Subject: [PATCH 1/6] add partition info in test --- tests/test_batchq.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_batchq.py b/tests/test_batchq.py index b99e1f3..bb75a90 100644 --- a/tests/test_batchq.py +++ b/tests/test_batchq.py @@ -11,6 +11,7 @@ def valid_job_submission() -> JobSubmission: return JobSubmission( jobstring="Hello World", + partition="xenon1t", qos="xenon1t", hours=10, container="xenonnt-development.simg", @@ -151,3 +152,5 @@ def test_submit_job_arguments(): assert ( len(missing_params) == 0 ), f"Missing parameters in submit_job: {', '.join(missing_params)}" + + \ No newline at end of file From 82a36ce65ceacd7c4c831f154dae83a257fcbbb5 Mon Sep 17 00:00:00 2001 From: yuema137 <3124558229@qq.com> Date: Fri, 19 Apr 2024 18:31:16 -0500 Subject: [PATCH 2/6] put bind in front of partition --- utilix/batchq.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utilix/batchq.py b/utilix/batchq.py index e72afd0..dd43ae2 100644 --- a/utilix/batchq.py +++ b/utilix/batchq.py @@ -123,6 +123,10 @@ class JobSubmission(BaseModel): False, description="Exclude the loosely coupled nodes" ) log: str = Field("job.log", description="Where to store the log file of the job") + bind: List[str] = Field( + default_factory=lambda: DEFAULT_BIND, + description="Paths to add to the container. Immutable when specifying dali as partition", + ) partition: Literal[ "dali", "lgrandi", "xenon1t", "broadwl", "kicp", "caslake", "build" ] = Field("xenon1t", description="Partition to submit the job to") @@ -137,10 +141,6 @@ class JobSubmission(BaseModel): container: str = Field( "xenonnt-development.simg", description="Name of the container to activate" ) - bind: List[str] = Field( - default_factory=lambda: DEFAULT_BIND, - description="Paths to add to the container. Immutable when specifying dali as partition", - ) cpus_per_task: int = Field(1, description="CPUs requested for job") hours: Optional[float] = Field(None, description="Max hours of a job") node: Optional[str] = Field( From f70d566ae4bc42945ee6362c4f996012ca2169ef Mon Sep 17 00:00:00 2001 From: yuema137 <3124558229@qq.com> Date: Thu, 13 Jun 2024 14:04:56 -0500 Subject: [PATCH 3/6] bump version -> v0.8.5 --- .bumpversion.cfg | 2 +- utilix/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 26ec642..29c7da1 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.8.4 +current_version = 0.8.5 files = setup.py utilix/__init__.py commit = True tag = True diff --git a/utilix/__init__.py b/utilix/__init__.py index e8c471b..2db640c 100644 --- a/utilix/__init__.py +++ b/utilix/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.8.4" +__version__ = "0.8.5" from . import config From 0b8c4f7288b9b32eb5359607095563bf04429e84 Mon Sep 17 00:00:00 2001 From: yuema137 <3124558229@qq.com> Date: Fri, 21 Jun 2024 21:23:52 -0500 Subject: [PATCH 4/6] add bigmem2 and gpu2 --- utilix/batchq.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/utilix/batchq.py b/utilix/batchq.py index d02435b..6512f92 100644 --- a/utilix/batchq.py +++ b/utilix/batchq.py @@ -32,6 +32,8 @@ "kicp", "caslake", "build", + "bigmem2", + "gpu2", ] TMPDIR: Dict[str, str] = { "dali": f"/dali/lgrandi/{USER}/tmp", @@ -41,6 +43,8 @@ "kicp": os.path.join(SCRATCH_DIR, "tmp"), "caslake": os.path.join(SCRATCH_DIR, "tmp"), "build": os.path.join(SCRATCH_DIR, "tmp"), + "bigmem2": os.path.join(SCRATCH_DIR, "tmp"), + "gpu2": os.path.join(SCRATCH_DIR, "tmp"), } SINGULARITY_DIR: str = "lgrandi/xenonnt/singularity-images" @@ -122,7 +126,7 @@ class JobSubmission(BaseModel): ) log: str = Field("job.log", description="Where to store the log file of the job") partition: Literal[ - "dali", "lgrandi", "xenon1t", "broadwl", "kicp", "caslake", "build" + "dali", "lgrandi", "xenon1t", "broadwl", "kicp", "caslake", "build", "bigmem2", "gpu2" ] = Field("xenon1t", description="Partition to submit the job to") bind: List[str] = Field( default_factory=lambda: DEFAULT_BIND, @@ -488,7 +492,7 @@ def submit_job( exclude_lc_nodes: bool = False, log: str = "job.log", partition: Literal[ - "dali", "lgrandi", "xenon1t", "broadwl", "kicp", "caslake", "build" + "dali", "lgrandi", "xenon1t", "broadwl", "kicp", "caslake", "build", "bigmem2", "gpu2" ] = "xenon1t", qos: str = "xenon1t", account: str = "pi-lgrandi", @@ -513,7 +517,7 @@ def submit_job( jobstring (str): The command to execute. exclude_lc_nodes (bool): Exclude the loosely coupled nodes. Default is True. log (str): Where to store the log file of the job. Default is "job.log". - partition (Literal["dali", "lgrandi", "xenon1t", "broadwl", "kicp", "caslake", "build"]): + partition (Literal["dali", "lgrandi", "xenon1t", "broadwl", "kicp", "caslake", "build", "bigmem2", "gpu2" (the only GPU node)]): Partition to submit the job to. Default is "xenon1t". qos (str): QOS to submit the job to. Default is "xenon1t". account (str): Account to submit the job to. Default is "pi-lgrandi". From 33f763f6868846a7862e35bbd4b6e7c5c57a9218 Mon Sep 17 00:00:00 2001 From: yuema137 <3124558229@qq.com> Date: Fri, 21 Jun 2024 21:33:30 -0500 Subject: [PATCH 5/6] update unnitest for batchq to validate all partitions --- tests/test_batchq.py | 83 ++++++++++++++++++++++++++++++-------------- 1 file changed, 57 insertions(+), 26 deletions(-) diff --git a/tests/test_batchq.py b/tests/test_batchq.py index 4dbc130..2cfafc1 100644 --- a/tests/test_batchq.py +++ b/tests/test_batchq.py @@ -29,16 +29,36 @@ def get_server_type(): def get_partition_and_qos(server): if server == "Midway2": - return "xenon1t", "xenon1t" + return [ + "xenon1t", + "broadwl", + "kicp", + "build", + "bigmem2", + "gpu2", + ] elif server == "Midway3": - return "lgrandi", "lgrandi" + return [ + "broadwl", + "kicp", + "lgrandi", + "caslake", + "build", + ] elif server == "Dali": - return "dali", "dali" + return [ + "dali", + "xenon1t", + "broadwl", + "kicp", + "build", + "bigmem2", + "gpu2", + ] else: raise ValueError(f"Unknown server: {server}") - -PARTITION, QOS = get_partition_and_qos(SERVER) +PARTITIONS = get_partition_and_qos(SERVER) # Fixture to provide a sample valid JobSubmission instance @@ -46,35 +66,46 @@ def get_partition_and_qos(server): def valid_job_submission() -> JobSubmission: return JobSubmission( jobstring="Hello World", - partition=PARTITION, - qos=QOS, + partition=PARTITIONS[0], + qos=PARTITIONS[0], hours=10, container="xenonnt-development.simg", ) -def test_job_submission_submit(valid_job_submission: JobSubmission): +import pytest + +@pytest.mark.parametrize("partition", PARTITIONS) +def test_job_submission_submit_all_partitions(partition): + job_submission = JobSubmission( + jobstring="echo 'Job started'; sleep 10; echo 'Job completed'", + partition=partition, + qos=partition, # QOS is the same as the partition + hours=1, + container="xenonnt-development.simg", + ) + with patch("utilix.batchq.Slurm") as mock_slurm_class: mock_slurm = MagicMock() mock_slurm_class.return_value = mock_slurm - valid_job_submission.jobstring = "echo 'Job started'; sleep 10; echo 'Job completed'" - valid_job_submission.submit() + job_submission.submit() mock_slurm_class.assert_called_once_with( - job_name=valid_job_submission.jobname, - output=valid_job_submission.log, - qos=valid_job_submission.qos, - error=valid_job_submission.log, - account=valid_job_submission.account, - partition=valid_job_submission.partition, - mem_per_cpu=valid_job_submission.mem_per_cpu, - cpus_per_task=valid_job_submission.cpus_per_task, - time=datetime.timedelta(hours=valid_job_submission.hours), + job_name=job_submission.jobname, + output=job_submission.log, + qos=partition, + error=job_submission.log, + account=job_submission.account, + partition=partition, + mem_per_cpu=job_submission.mem_per_cpu, + cpus_per_task=job_submission.cpus_per_task, + time=datetime.timedelta(hours=job_submission.hours), ) mock_slurm.add_cmd.assert_called_once() mock_slurm.sbatch.assert_called_once_with(shell="/bin/bash") -def test_submit_job_function(): +@pytest.mark.parametrize("partition", PARTITIONS) +def test_submit_job_function_all_partitions(partition): jobstring = "echo 'Job started'; sleep 10; echo 'Job completed'" with patch("utilix.batchq.JobSubmission") as mock_job_submission_class: @@ -83,9 +114,9 @@ def test_submit_job_function(): submit_job( jobstring=jobstring, - partition=PARTITION, - qos=QOS, - hours=10, + partition=partition, + qos=partition, + hours=1, container="xenonnt-development.simg", ) @@ -93,8 +124,8 @@ def test_submit_job_function(): jobstring=jobstring, exclude_lc_nodes=False, log="job.log", - partition=PARTITION, - qos=QOS, + partition=partition, + qos=partition, account="pi-lgrandi", jobname="somejob", sbatch_file=None, @@ -103,7 +134,7 @@ def test_submit_job_function(): container="xenonnt-development.simg", bind=batchq.DEFAULT_BIND, cpus_per_task=1, - hours=10, + hours=1, node=None, exclude_nodes=None, dependency=None, From 48dbdc0e9700c76076e89a52ee045fcc537fd21f Mon Sep 17 00:00:00 2001 From: yuema137 <3124558229@qq.com> Date: Fri, 21 Jun 2024 21:39:56 -0500 Subject: [PATCH 6/6] remove broadwl for midway3 --- tests/test_batchq.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_batchq.py b/tests/test_batchq.py index 2cfafc1..d208c70 100644 --- a/tests/test_batchq.py +++ b/tests/test_batchq.py @@ -39,7 +39,6 @@ def get_partition_and_qos(server): ] elif server == "Midway3": return [ - "broadwl", "kicp", "lgrandi", "caslake",