From 3cedc38df533a5b0f7477a6c1d3439decbf94c5d Mon Sep 17 00:00:00 2001 From: dbeltran Date: Tue, 10 Dec 2024 14:52:09 +0100 Subject: [PATCH] Added notify_on --- autosubmit/job/job.py | 17 ++++++- autosubmit/notifications/mail_notifier.py | 5 +- docs/source/userguide/configure/index.rst | 17 ++++++- setup.py | 2 +- test/unit/test_job_pytest.py | 58 ++++++++++++++++++----- 5 files changed, 80 insertions(+), 19 deletions(-) diff --git a/autosubmit/job/job.py b/autosubmit/job/job.py index b6543d08a..0a5082323 100644 --- a/autosubmit/job/job.py +++ b/autosubmit/job/job.py @@ -568,6 +568,16 @@ def splits(self): def splits(self, value): self._splits = value + @property + @autosubmit_parameter(name='notify_on') + def notify_on(self): + """Send mail notification on job status change.""" + return self._notify_on + + @notify_on.setter + def notify_on(self, value): + self._notify_on = value + def __getstate__(self): return {k: v for k, v in self.__dict__.items() if k not in ["_platform", "_children", "_parents", "submitter"]} @@ -1943,7 +1953,12 @@ def update_job_parameters(self,as_conf, parameters): self.shape = as_conf.jobs_data[self.section].get("SHAPE", "") self.script = as_conf.jobs_data[self.section].get("SCRIPT", "") self.x11 = False if str(as_conf.jobs_data[self.section].get("X11", False)).lower() == "false" else True - self.update_stat_file() + self.notify_on = as_conf.jobs_data[self.section].get("NOTIFY_ON", []) + if self.wrapper_type != "vertical" and self.packed: + self.stat_file = f"{self.script_name[:-4]}_STAT_{self.fail_count}" + else: + self.stat_file = f"{self.script_name[:-4]}_STAT_0" + if self.checkpoint: # To activate placeholder sustitution per in the template parameters["AS_CHECKPOINT"] = self.checkpoint parameters['JOBNAME'] = self.name diff --git a/autosubmit/notifications/mail_notifier.py b/autosubmit/notifications/mail_notifier.py index 45c01c1c9..e6a5f2c43 100644 --- a/autosubmit/notifications/mail_notifier.py +++ b/autosubmit/notifications/mail_notifier.py @@ -38,13 +38,14 @@ def notify_experiment_status(self, exp_id,mail_to,platform): self._send_mail(self.config.MAIL_FROM, mail, message) except BaseException as e: Log.printlog('An error has occurred while sending a mail for warn about remote_platform', 6011) + def notify_status_change(self, exp_id, job_name, prev_status, status, mail_to): message_text = self._generate_message_text(exp_id, job_name, prev_status, status) message = MIMEText(message_text) message['From'] = email.utils.formataddr(('Autosubmit', self.config.MAIL_FROM)) message['Subject'] = f'[Autosubmit] The job {job_name} status has changed to {str(status)}' message['Date'] = email.utils.formatdate(localtime=True) - for mail in mail_to: + for mail in mail_to: # expects a list message['To'] = email.utils.formataddr((mail, mail)) try: self._send_mail(self.config.MAIL_FROM, mail, message) @@ -74,4 +75,4 @@ def _generate_message_experiment_status(exp_id, platform=""): + f'Platform affected:{str(platform.name)} using as host:{str(platform.host)} \n\n' \ f'[WARN] Autosubmit encountered an issue with an remote_platform.\n It will resume itself, whenever is possible\n If issue persist, you can change the host IP or put multiple hosts in the platform.yml' + '\n\n\n\n\n' \ f'INFO: This message was auto generated by Autosubmit, '\ - f'remember that you can disable these messages on Autosubmit config file. \n' \ No newline at end of file + f'remember that you can disable these messages on Autosubmit config file. \n' diff --git a/docs/source/userguide/configure/index.rst b/docs/source/userguide/configure/index.rst index ecbcce89d..37f1c76c5 100644 --- a/docs/source/userguide/configure/index.rst +++ b/docs/source/userguide/configure/index.rst @@ -221,7 +221,10 @@ Example: # Default: False NOTIFICATIONS: True # Mail address where notifications will be received - TO: jsmith@example.com rlewis@example.com + TO: + - jsmith@example.com + - rlewis@example.com + 2. Then you have to define for which jobs you want to be notified. @@ -232,7 +235,7 @@ Edit ``jobs_cxxx.yml`` in the ``conf`` folder of the experiment. defined on the parameter ``NOTIFY_ON`` .. hint:: - Remember that you can define more than one job status divided by a whitespace. +Remember that you can define more than one job status separated by a whitespace, a comma (`,`), or using a list. Example: :: @@ -246,6 +249,16 @@ Example: FILE: LOCAL_SETUP.sh PLATFORM: LOCAL NOTIFY_ON: FAILED COMPLETED + EXAMPLE_JOB: + FILE: EXAMPLE_JOB.sh + PLATFORM: LOCAL + NOTIFY_ON: FAILED, COMPLETED + EXAMPLE_JOB_2: + FILE: EXAMPLE_JOB_2.sh + PLATFORM: LOCAL + NOTIFY_ON: + - FAILED + - COMPLETED How to add a new platform ------------------------- diff --git a/setup.py b/setup.py index 807ed719a..3cee16425 100644 --- a/setup.py +++ b/setup.py @@ -44,7 +44,7 @@ 'py3dotplus==1.1.0', 'numpy<2', 'rocrate==0.*', - 'autosubmitconfigparser==1.0.75', + 'autosubmitconfigparser==1.0.76', 'configparser', 'setproctitle', 'invoke>=2.0', diff --git a/test/unit/test_job_pytest.py b/test/unit/test_job_pytest.py index 71e2db21e..eed86837c 100644 --- a/test/unit/test_job_pytest.py +++ b/test/unit/test_job_pytest.py @@ -6,13 +6,29 @@ from pathlib import Path +def create_job_and_update_parameters(autosubmit_config, experiment_data): + as_conf = autosubmit_config("test-expid", experiment_data) + as_conf.experiment_data = as_conf.deep_normalize(as_conf.experiment_data) + as_conf.experiment_data = as_conf.normalize_variables(as_conf.experiment_data, must_exists=True) + as_conf.experiment_data = as_conf.deep_read_loops(as_conf.experiment_data) + as_conf.experiment_data = as_conf.substitute_dynamic_variables(as_conf.experiment_data) + as_conf.experiment_data = as_conf.parse_data_loops(as_conf.experiment_data) + # Create some jobs + job = Job('A', '1', 0, 1) + platform = PsPlatform(expid='a000', name='DUMMY_PLATFORM', config=as_conf.experiment_data) + job.section = 'RANDOM-SECTION' + job.platform = platform + job.update_parameters(as_conf, {}) + return job + + @pytest.mark.parametrize('experiment_data, expected_data', [( { 'JOBS': { 'RANDOM-SECTION': { 'FILE': "test.sh", 'PLATFORM': 'DUMMY_PLATFORM', - 'TEST': "%other%" + 'TEST': "%other%", }, }, 'PLATFORMS': { @@ -38,18 +54,7 @@ } )]) def test_update_parameters_current_variables(autosubmit_config, experiment_data, expected_data): - as_conf = autosubmit_config("test-expid", experiment_data) - as_conf.experiment_data = as_conf.deep_normalize(as_conf.experiment_data) - as_conf.experiment_data = as_conf.normalize_variables(as_conf.experiment_data, must_exists=True) - as_conf.experiment_data = as_conf.deep_read_loops(as_conf.experiment_data) - as_conf.experiment_data = as_conf.substitute_dynamic_variables(as_conf.experiment_data) - as_conf.experiment_data = as_conf.parse_data_loops(as_conf.experiment_data) - # Create some jobs - job = Job('A', '1', 0, 1) - platform = PsPlatform(expid='a000', name='DUMMY_PLATFORM', config=as_conf.experiment_data) - job.section = 'RANDOM-SECTION' - job.platform = platform - job.update_parameters(as_conf, {}) + job = create_job_and_update_parameters(autosubmit_config, experiment_data) for key, value in expected_data.items(): assert job.parameters[key] == value @@ -116,3 +121,30 @@ def test_recover_last_log_name(tmpdir, test_with_logfiles, file_timestamp_greate assert job.updated_log == expected_update_log assert job.local_logs[0] == str(expected_local_logs[0]) assert job.local_logs[1] == str(expected_local_logs[1]) + + +@pytest.mark.parametrize('experiment_data, attributes_to_check', [( + { + 'JOBS': { + 'RANDOM-SECTION': { + 'FILE': "test.sh", + 'PLATFORM': 'DUMMY_PLATFORM', + 'NOTIFY_ON': 'COMPLETED', + }, + }, + 'PLATFORMS': { + 'dummy_platform': { + 'type': 'ps', + }, + }, + 'ROOTDIR': 'dummy_rootdir', + 'LOCAL_TMP_DIR': 'dummy_tmpdir', + 'LOCAL_ROOT_DIR': 'dummy_rootdir', + }, + {'notify_on': ['COMPLETED']} +)]) +def test_update_parameters_attributes(autosubmit_config, experiment_data, attributes_to_check): + job = create_job_and_update_parameters(autosubmit_config, experiment_data) + for attr in attributes_to_check: + assert hasattr(job, attr) + assert getattr(job, attr) == attributes_to_check[attr]