From 64bcbe2e83e572f583e4c1ed90be18a056ee86a5 Mon Sep 17 00:00:00 2001 From: yuema137 <3124558229@qq.com> Date: Fri, 19 Apr 2024 18:30:28 -0500 Subject: [PATCH 1/6] add partition info in test --- tests/test_batchq.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/test_batchq.py b/tests/test_batchq.py index b99e1f3..bb75a90 100644 --- a/tests/test_batchq.py +++ b/tests/test_batchq.py @@ -11,6 +11,7 @@ def valid_job_submission() -> JobSubmission: return JobSubmission( jobstring="Hello World", + partition="xenon1t", qos="xenon1t", hours=10, container="xenonnt-development.simg", @@ -151,3 +152,5 @@ def test_submit_job_arguments(): assert ( len(missing_params) == 0 ), f"Missing parameters in submit_job: {', '.join(missing_params)}" + + \ No newline at end of file From 82a36ce65ceacd7c4c831f154dae83a257fcbbb5 Mon Sep 17 00:00:00 2001 From: yuema137 <3124558229@qq.com> Date: Fri, 19 Apr 2024 18:31:16 -0500 Subject: [PATCH 2/6] put bind in front of partition --- utilix/batchq.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/utilix/batchq.py b/utilix/batchq.py index e72afd0..dd43ae2 100644 --- a/utilix/batchq.py +++ b/utilix/batchq.py @@ -123,6 +123,10 @@ class JobSubmission(BaseModel): False, description="Exclude the loosely coupled nodes" ) log: str = Field("job.log", description="Where to store the log file of the job") + bind: List[str] = Field( + default_factory=lambda: DEFAULT_BIND, + description="Paths to add to the container. Immutable when specifying dali as partition", + ) partition: Literal[ "dali", "lgrandi", "xenon1t", "broadwl", "kicp", "caslake", "build" ] = Field("xenon1t", description="Partition to submit the job to") @@ -137,10 +141,6 @@ class JobSubmission(BaseModel): container: str = Field( "xenonnt-development.simg", description="Name of the container to activate" ) - bind: List[str] = Field( - default_factory=lambda: DEFAULT_BIND, - description="Paths to add to the container. Immutable when specifying dali as partition", - ) cpus_per_task: int = Field(1, description="CPUs requested for job") hours: Optional[float] = Field(None, description="Max hours of a job") node: Optional[str] = Field( From 5ec6638d8540b4e121e386a9e360fd35492b9431 Mon Sep 17 00:00:00 2001 From: yuema137 <3124558229@qq.com> Date: Thu, 30 May 2024 14:04:44 -0500 Subject: [PATCH 3/6] generate different test template for each server --- tests/test_batchq.py | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/tests/test_batchq.py b/tests/test_batchq.py index 282f104..0c2861c 100644 --- a/tests/test_batchq.py +++ b/tests/test_batchq.py @@ -2,17 +2,39 @@ from utilix import batchq from utilix.batchq import JobSubmission, QOSNotFoundError, FormatError, submit_job import pytest +import os from unittest.mock import patch import inspect +# Get the server type +def get_server_type(): + hostname = os.uname().nodename + if "midway2" in hostname: + return "Midway2" + elif "midway3" in hostname: + return "Midway3" + elif "dali" in hostname: + return "Dali" + else: + raise ValueError(f"Unknown server type for hostname {hostname}. Please use midway2, midway3, or dali.") # Fixture to provide a sample valid JobSubmission instance @pytest.fixture def valid_job_submission() -> JobSubmission: + server = get_server_type() + if server == "Midway2": + partition = "xenon1t" + qos = "xenon1t" + elif server == "Midway3": + partition = "lgrandi" + qos = "lgrandi" + elif server == "Dali": + partition = "dali" + qos = "dali" return JobSubmission( jobstring="Hello World", - partition="xenon1t", - qos="xenon1t", + partition=partition, + qos=qos, hours=10, container="xenonnt-development.simg", ) From 1771077079c7c41fb42a2ba0203c13566a6ef1db Mon Sep 17 00:00:00 2001 From: yuema137 <3124558229@qq.com> Date: Thu, 30 May 2024 14:15:13 -0500 Subject: [PATCH 4/6] add more helpers --- tests/test_batchq.py | 54 ++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/tests/test_batchq.py b/tests/test_batchq.py index 0c2861c..f0e7569 100644 --- a/tests/test_batchq.py +++ b/tests/test_batchq.py @@ -6,7 +6,8 @@ from unittest.mock import patch import inspect -# Get the server type + +# Get the SERVER type def get_server_type(): hostname = os.uname().nodename if "midway2" in hostname: @@ -16,25 +17,35 @@ def get_server_type(): elif "dali" in hostname: return "Dali" else: - raise ValueError(f"Unknown server type for hostname {hostname}. Please use midway2, midway3, or dali.") + raise ValueError( + f"Unknown server type for hostname {hostname}. Please use midway2, midway3, or dali." + ) -# Fixture to provide a sample valid JobSubmission instance -@pytest.fixture -def valid_job_submission() -> JobSubmission: - server = get_server_type() + +SERVER = get_server_type() + + +def get_partition_and_qos(server): if server == "Midway2": - partition = "xenon1t" - qos = "xenon1t" + return "xenon1t", "xenon1t" elif server == "Midway3": - partition = "lgrandi" - qos = "lgrandi" + return "lgrandi", "lgrandi" elif server == "Dali": - partition = "dali" - qos = "dali" + return "dali", "dali" + else: + raise ValueError(f"Unknown server: {server}") + + +PARTITION, QOS = get_partition_and_qos(SERVER) + + +# Fixture to provide a sample valid JobSubmission instance +@pytest.fixture +def valid_job_submission() -> JobSubmission: return JobSubmission( jobstring="Hello World", - partition=partition, - qos=qos, + partition=PARTITION, + qos=QOS, hours=10, container="xenonnt-development.simg", ) @@ -59,7 +70,7 @@ def test_invalid_qos(): def test_valid_qos(valid_job_submission: JobSubmission): """Test case to check if a valid qos is accepted.""" - assert valid_job_submission.qos == "xenon1t" + assert valid_job_submission.qos == valid_job_submission.qos def test_invalid_hours(): @@ -67,7 +78,8 @@ def test_invalid_hours(): with pytest.raises(ValidationError) as exc_info: JobSubmission( jobstring="Hello World", - qos="xenon1t", + partition=PARTITION, + qos=QOS, hours=100, container="xenonnt-development.simg", ) @@ -76,14 +88,18 @@ def test_invalid_hours(): def test_valid_hours(valid_job_submission: JobSubmission): """Test case to check if a valid hours value is accepted.""" - assert valid_job_submission.hours == 10 + assert valid_job_submission.hours == valid_job_submission.hours def test_invalid_container(): """Test case to check if the appropriate validation error is raised when an invalid value is provided for the container field.""" with pytest.raises(FormatError) as exc_info: JobSubmission( - jobstring="Hello World", qos="xenon1t", hours=10, container="invalid.ext" + jobstring="Hello World", + partition=PARTITION, + qos=QOS, + hours=10, + container="invalid.ext", ) assert "Container must end with .simg" in str(exc_info.value) @@ -178,5 +194,3 @@ def test_submit_job_arguments(): assert ( len(missing_params) == 0 ), f"Missing parameters in submit_job: {', '.join(missing_params)}" - - \ No newline at end of file From 4da75aa4c2c8040423e35daeac74da7988a71fda Mon Sep 17 00:00:00 2001 From: yuema137 <3124558229@qq.com> Date: Thu, 30 May 2024 14:59:22 -0500 Subject: [PATCH 5/6] add invalid binding test --- tests/test_batchq.py | 96 ++++++++++++++++++++++---------------------- 1 file changed, 49 insertions(+), 47 deletions(-) diff --git a/tests/test_batchq.py b/tests/test_batchq.py index f0e7569..9e86b2c 100644 --- a/tests/test_batchq.py +++ b/tests/test_batchq.py @@ -5,6 +5,7 @@ import os from unittest.mock import patch import inspect +import logging # Get the SERVER type @@ -56,15 +57,40 @@ def test_valid_jobstring(valid_job_submission: JobSubmission): assert valid_job_submission.jobstring == "Hello World" -def test_invalid_qos(): +def test_valid_container(valid_job_submission: JobSubmission): + """Test case to check if a valid path for the container is found.""" + assert "xenonnt-development.simg" in valid_job_submission.container + + +def test_container_exists(valid_job_submission: JobSubmission, tmp_path: str): + """ + Test case to check if the appropriate validation error is raised when the specified container does not exist. + """ + invalid_container = "nonexistent-container.simg" + with patch.object(batchq, "SINGULARITY_DIR", "/lgrandi/xenonnt/singularity-images"): + with pytest.raises(FileNotFoundError) as exc_info: + JobSubmission( + **valid_job_submission.dict(exclude={"container"}), + container=invalid_container, + ) + assert f"Container {invalid_container} does not exist" in str(exc_info.value) + + +def test_invalid_container(valid_job_submission: JobSubmission): + """Test case to check if the appropriate validation error is raised when an invalid value is provided for the container field.""" + job_submission_data = valid_job_submission.dict().copy() + job_submission_data["container"] = "invalid.txt" + with pytest.raises(FormatError) as exc_info: + job_submission = JobSubmission(**job_submission_data) + assert "Container must end with .simg" in str(exc_info.value) + + +def test_invalid_qos(valid_job_submission: JobSubmission): """Test case to check if the appropriate validation error is raised when an invalid value is provided for the qos field.""" + job_submission_data = valid_job_submission.dict().copy() + job_submission_data["qos"] = "invalid_qos" with pytest.raises(QOSNotFoundError) as exc_info: - JobSubmission( - jobstring="Hello World", - qos="invalid_qos", - hours=10, - container="xenonnt-development.simg", - ) + JobSubmission(**job_submission_data) assert "QOS invalid_qos is not in the list of available qos" in str(exc_info.value) @@ -73,16 +99,24 @@ def test_valid_qos(valid_job_submission: JobSubmission): assert valid_job_submission.qos == valid_job_submission.qos -def test_invalid_hours(): +def test_invalid_bind(valid_job_submission: JobSubmission, caplog): + """Test case to check if the appropriate validation error is raised when an invalid value is provided for the bind field.""" + job_submission_data = valid_job_submission.dict().copy() + invalid_bind = "/project999" + job_submission_data["bind"].append(invalid_bind) + with caplog.at_level(logging.WARNING): + JobSubmission(**job_submission_data) + + assert "skipped mounting" in caplog.text + assert invalid_bind in caplog.text + + +def test_invalid_hours(valid_job_submission: JobSubmission): """Test case to check if the appropriate validation error is raised when an invalid value is provided for the hours field.""" + job_submission_data = valid_job_submission.dict().copy() + job_submission_data["hours"] = 1000 with pytest.raises(ValidationError) as exc_info: - JobSubmission( - jobstring="Hello World", - partition=PARTITION, - qos=QOS, - hours=100, - container="xenonnt-development.simg", - ) + JobSubmission(**job_submission_data) assert "Hours must be between 0 and 72" in str(exc_info.value) @@ -91,38 +125,6 @@ def test_valid_hours(valid_job_submission: JobSubmission): assert valid_job_submission.hours == valid_job_submission.hours -def test_invalid_container(): - """Test case to check if the appropriate validation error is raised when an invalid value is provided for the container field.""" - with pytest.raises(FormatError) as exc_info: - JobSubmission( - jobstring="Hello World", - partition=PARTITION, - qos=QOS, - hours=10, - container="invalid.ext", - ) - assert "Container must end with .simg" in str(exc_info.value) - - -def test_valid_container(valid_job_submission: JobSubmission): - """Test case to check if a valid path for the container is found.""" - assert "xenonnt-development.simg" in valid_job_submission.container - - -def test_container_exists(valid_job_submission: JobSubmission, tmp_path: str): - """ - Test case to check if the appropriate validation error is raised when the specified container does not exist. - """ - invalid_container = "nonexistent-container.simg" - with patch.object(batchq, "SINGULARITY_DIR", "/lgrandi/xenonnt/singularity-images"): - with pytest.raises(FileNotFoundError) as exc_info: - JobSubmission( - **valid_job_submission.dict(exclude={"container"}), - container=invalid_container, - ) - assert f"Container {invalid_container} does not exist" in str(exc_info.value) - - def test_bypass_validation_qos(valid_job_submission: JobSubmission): """ Test case to check if the validation for the qos field is skipped when it is included in the bypass_validation list. From 44767d317e60a349e63f6836a07777fc53b731d7 Mon Sep 17 00:00:00 2001 From: yuema137 <3124558229@qq.com> Date: Thu, 30 May 2024 14:59:47 -0500 Subject: [PATCH 6/6] update check_bind validation --- utilix/batchq.py | 32 ++++++++++++++++++++++++-------- 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/utilix/batchq.py b/utilix/batchq.py index 7e026c1..d02435b 100644 --- a/utilix/batchq.py +++ b/utilix/batchq.py @@ -58,15 +58,19 @@ "/dali/lgrandi/grid_proxy/xenon_service_proxy:/project2/lgrandi/grid_proxy/xenon_service_proxy", ] + class QOSNotFoundError(Exception): """ Provided qos is not found in the qos list """ + + class FormatError(Exception): """ Format of file is not correct """ + def _make_executable(path: str) -> None: """ Make a file executable by the user, group and others. @@ -103,6 +107,7 @@ def _get_qos_list() -> List[str]: print(f"An error occurred while executing sacctmgr: {e}") return [] + class JobSubmission(BaseModel): """ Class to generate and submit a job to the SLURM queue. @@ -116,13 +121,13 @@ class JobSubmission(BaseModel): False, description="Exclude the loosely coupled nodes" ) log: str = Field("job.log", description="Where to store the log file of the job") + partition: Literal[ + "dali", "lgrandi", "xenon1t", "broadwl", "kicp", "caslake", "build" + ] = Field("xenon1t", description="Partition to submit the job to") bind: List[str] = Field( default_factory=lambda: DEFAULT_BIND, description="Paths to add to the container. Immutable when specifying dali as partition", ) - partition: Literal[ - "dali", "lgrandi", "xenon1t", "broadwl", "kicp", "caslake", "build" - ] = Field("xenon1t", description="Partition to submit the job to") qos: str = Field("xenon1t", description="QOS to submit the job to") account: str = Field("pi-lgrandi", description="Account to submit the job to") jobname: str = Field("somejob", description="How to name this job") @@ -175,12 +180,13 @@ def _skip_validation(cls, field: str, values: Dict[Any, Any]) -> bool: bool: True if the field should be validated, False otherwise. """ return field in values.get("bypass_validation", []) + # validate the bypass_validation so that it can be reached in values @validator("bypass_validation", pre=True, each_item=True) def check_bypass_validation(cls, v: list) -> list: return v - @validator("bind", pre=True, each_item=True) + @validator("bind", pre=True) def check_bind(cls, v: str, values: Dict[Any, Any]) -> str: """ Check if the bind path exists. @@ -194,10 +200,20 @@ def check_bind(cls, v: str, values: Dict[Any, Any]) -> str: if cls._skip_validation("bind", values): return v - if not os.path.exists(v): - logger.warning("Bind path %s does not exist", v) - - return v + valid_bind = [] + invalid_bind = [] + for path in v: + if ":" in path: + actual_path = path.split(":")[0] + else: + actual_path = path + if os.path.exists(actual_path): + valid_bind.append(path) + else: + invalid_bind.append(path) + if len(invalid_bind) > 0: + logger.warning("Invalid bind paths: %s, skipped mounting", invalid_bind) + return valid_bind @validator("partition", pre=True, always=True) def overwrite_for_dali(cls, v: str, values: Dict[Any, Any]) -> str: