From bc69b825f4fc62571d7ec311482ca7f7d0c00c2d Mon Sep 17 00:00:00 2001 From: burnout87 Date: Wed, 11 Oct 2023 18:33:48 +0200 Subject: [PATCH 01/23] return_progress as arg for run_query --- cdci_data_analysis/analysis/instrument.py | 2 ++ cdci_data_analysis/analysis/queries.py | 3 ++- cdci_data_analysis/flask_app/dispatcher_query.py | 2 ++ 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cdci_data_analysis/analysis/instrument.py b/cdci_data_analysis/analysis/instrument.py index 8603767b2..296999cd5 100644 --- a/cdci_data_analysis/analysis/instrument.py +++ b/cdci_data_analysis/analysis/instrument.py @@ -390,6 +390,7 @@ def run_query(self, product_type, dry_run=False, api=False, decoded_token=None, + return_progress=False, **kwargs): if logger is None: @@ -433,6 +434,7 @@ def run_query(self, product_type, config=config, logger=logger, sentry_dsn=sentry_dsn, + report_progress=return_progress, api=api) if query_out.status_dictionary['status'] == 0: if 'comment' in query_out.status_dictionary.keys(): diff --git a/cdci_data_analysis/analysis/queries.py b/cdci_data_analysis/analysis/queries.py index d3566461b..4e2beb554 100644 --- a/cdci_data_analysis/analysis/queries.py +++ b/cdci_data_analysis/analysis/queries.py @@ -677,7 +677,8 @@ def run_query(self, config=None, logger=None, sentry_dsn=None, - api=False): + api=False, + return_progress=False): # print ('--> running query for ',instrument.name,'with config',config) if logger is None: diff --git a/cdci_data_analysis/flask_app/dispatcher_query.py b/cdci_data_analysis/flask_app/dispatcher_query.py index 7b28485a3..1496f31ce 100644 --- a/cdci_data_analysis/flask_app/dispatcher_query.py +++ b/cdci_data_analysis/flask_app/dispatcher_query.py @@ -161,6 +161,7 @@ def __init__(self, app, self.client_name = self.par_dic.pop('client-name', 'unknown') + self.return_progress = self.par_dic.pop('return-progress', False) == 'True' if os.environ.get("DISPATCHER_ASYNC_ENABLED", "no") == "yes": # TODO: move to config! self.async_dispatcher = self.par_dic.pop( 'async_dispatcher', 'True') == 'True' # why string true?? else false anyway @@ -1825,6 +1826,7 @@ def run_query(self, off_line=False, disp_conf=None): verbose=verbose, dry_run=dry_run, api=api, + return_progress=self.return_progress, decoded_token=self.decoded_token) self.log_query_progression("after instrument.run_query") except RequestNotAuthorized as e: From ed296f9c7bf8e86c47b45b1b0bc5e7c8dcc69833 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 12 Oct 2023 14:54:22 +0200 Subject: [PATCH 02/23] cc-like receivers room ids configurable --- cdci_data_analysis/analysis/matrix_helper.py | 12 +++++++++++- cdci_data_analysis/config_dir/conf_env.yml.example | 2 ++ cdci_data_analysis/configurer.py | 3 +++ cdci_data_analysis/pytest_fixtures.py | 1 + 4 files changed, 17 insertions(+), 1 deletion(-) diff --git a/cdci_data_analysis/analysis/matrix_helper.py b/cdci_data_analysis/analysis/matrix_helper.py index 92ce22005..b6d98d56c 100644 --- a/cdci_data_analysis/analysis/matrix_helper.py +++ b/cdci_data_analysis/analysis/matrix_helper.py @@ -188,6 +188,8 @@ def send_job_message( matrix_sender_access_token = config.matrix_sender_access_token receiver_room_id = tokenHelper.get_token_user_matrix_room_id(decoded_token) + cc_receivers_room_id = config.matrix_server_url + matrix_message_data = { 'oda_site': { 'site_name': config.site_name, @@ -219,8 +221,16 @@ def send_job_message( message_body_html = template.render(**matrix_message_data) message_text = textify_matrix_message(message_body_html) res_data = send_message(url_server=matrix_server_url, + sender_access_token=matrix_sender_access_token, + room_id=receiver_room_id, + message_text=message_text, + message_body_html=message_body_html + ) + + for cc_room_id in cc_receivers_room_id: + res_data = send_message(url_server=matrix_server_url, sender_access_token=matrix_sender_access_token, - room_id=receiver_room_id, + room_id=cc_room_id, message_text=message_text, message_body_html=message_body_html ) diff --git a/cdci_data_analysis/config_dir/conf_env.yml.example b/cdci_data_analysis/config_dir/conf_env.yml.example index 467c7c120..65846701d 100644 --- a/cdci_data_analysis/config_dir/conf_env.yml.example +++ b/cdci_data_analysis/config_dir/conf_env.yml.example @@ -80,6 +80,8 @@ dispatcher: matrix_server_url: MATRIX_SERVER_URL # access token of the sender account, from which the messages will be sent from matrix_sender_access_token: MATRIX_SENDER_ACCESS_TOKEN + # list of room ids to which the matrix message should be sent + matrix_cc_receivers_room_id: [] # enable/disable sending of messages via matrix in case of a submitted job status matrix_message_sending_job_submitted: True # amount of time (in seconds) elapsed from the sending of the last message on matrix diff --git a/cdci_data_analysis/configurer.py b/cdci_data_analysis/configurer.py index 992badc5e..0fa8ce179 100644 --- a/cdci_data_analysis/configurer.py +++ b/cdci_data_analysis/configurer.py @@ -247,6 +247,7 @@ def __init__(self, cfg_dict, origin=None): disp_dict['email_options'].get('incident_report_email_options', {}).get('incident_report_receivers_email_addresses', None), disp_dict.get('matrix_options', {}).get('matrix_server_url', None), disp_dict.get('matrix_options', {}).get('matrix_sender_access_token', None), + disp_dict.get('matrix_options', {}).get('matrix_cc_receivers_room_id', []), disp_dict.get('matrix_options', {}).get('matrix_message_sending_job_submitted', True), disp_dict.get('matrix_options', {}).get('matrix_message_sending_job_submitted_default_interval', 1800), disp_dict.get('matrix_options', {}).get('sentry_for_matrix_message_sending_check', False), @@ -322,6 +323,7 @@ def set_conf_dispatcher(self, incident_report_receivers_email_addresses, matrix_server_url, matrix_sender_access_token, + matrix_cc_receivers_room_id, matrix_message_sending_job_submitted, matrix_message_sending_job_submitted_default_interval, sentry_for_matrix_message_sending_check, @@ -369,6 +371,7 @@ def set_conf_dispatcher(self, self.incident_report_receivers_email_addresses = incident_report_receivers_email_addresses self.matrix_server_url = matrix_server_url self.matrix_sender_access_token = matrix_sender_access_token + self.matrix_cc_receivers_room_id = matrix_cc_receivers_room_id self.matrix_message_sending_job_submitted = matrix_message_sending_job_submitted self.matrix_message_sending_job_submitted_default_interval = matrix_message_sending_job_submitted_default_interval self.sentry_for_matrix_message_sending_check = sentry_for_matrix_message_sending_check diff --git a/cdci_data_analysis/pytest_fixtures.py b/cdci_data_analysis/pytest_fixtures.py index 3f1a4c5fc..fa5f35faf 100644 --- a/cdci_data_analysis/pytest_fixtures.py +++ b/cdci_data_analysis/pytest_fixtures.py @@ -564,6 +564,7 @@ def dispatcher_test_conf_with_matrix_options_fn(dispatcher_test_conf_fn): f.write('\n matrix_options:' '\n matrix_server_url: "https://matrix-client.matrix.org/"' f'\n matrix_sender_access_token: "{os.getenv("MATRIX_SENDER_ACCESS_TOKEN", "matrix_sender_access_token")}"' + '\n matrix_cc_receivers_room_id: []' '\n matrix_message_sending_job_submitted: True' '\n matrix_message_sending_job_submitted_default_interval: 5' '\n sentry_for_matrix_message_sending_check: False' From a886b0dd83046985bde8c62be8ff1fc6ab9e00ed Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 12 Oct 2023 15:02:44 +0200 Subject: [PATCH 03/23] wrong arg --- cdci_data_analysis/analysis/instrument.py | 2 -- cdci_data_analysis/analysis/queries.py | 3 +-- cdci_data_analysis/flask_app/dispatcher_query.py | 2 -- 3 files changed, 1 insertion(+), 6 deletions(-) diff --git a/cdci_data_analysis/analysis/instrument.py b/cdci_data_analysis/analysis/instrument.py index 296999cd5..8603767b2 100644 --- a/cdci_data_analysis/analysis/instrument.py +++ b/cdci_data_analysis/analysis/instrument.py @@ -390,7 +390,6 @@ def run_query(self, product_type, dry_run=False, api=False, decoded_token=None, - return_progress=False, **kwargs): if logger is None: @@ -434,7 +433,6 @@ def run_query(self, product_type, config=config, logger=logger, sentry_dsn=sentry_dsn, - report_progress=return_progress, api=api) if query_out.status_dictionary['status'] == 0: if 'comment' in query_out.status_dictionary.keys(): diff --git a/cdci_data_analysis/analysis/queries.py b/cdci_data_analysis/analysis/queries.py index 4e2beb554..d3566461b 100644 --- a/cdci_data_analysis/analysis/queries.py +++ b/cdci_data_analysis/analysis/queries.py @@ -677,8 +677,7 @@ def run_query(self, config=None, logger=None, sentry_dsn=None, - api=False, - return_progress=False): + api=False): # print ('--> running query for ',instrument.name,'with config',config) if logger is None: diff --git a/cdci_data_analysis/flask_app/dispatcher_query.py b/cdci_data_analysis/flask_app/dispatcher_query.py index 60dcb6343..d498041e7 100644 --- a/cdci_data_analysis/flask_app/dispatcher_query.py +++ b/cdci_data_analysis/flask_app/dispatcher_query.py @@ -161,7 +161,6 @@ def __init__(self, app, self.client_name = self.par_dic.pop('client-name', 'unknown') - self.return_progress = self.par_dic.pop('return-progress', False) == 'True' if os.environ.get("DISPATCHER_ASYNC_ENABLED", "no") == "yes": # TODO: move to config! self.async_dispatcher = self.par_dic.pop( 'async_dispatcher', 'True') == 'True' # why string true?? else false anyway @@ -1910,7 +1909,6 @@ def run_query(self, off_line=False, disp_conf=None): verbose=verbose, dry_run=dry_run, api=api, - return_progress=self.return_progress, decoded_token=self.decoded_token) self.log_query_progression("after instrument.run_query") except RequestNotAuthorized as e: From 0cc683fd3cc4406daa00deb3daf864328aa44bab Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 12 Oct 2023 15:12:43 +0200 Subject: [PATCH 04/23] fix typo --- cdci_data_analysis/analysis/matrix_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cdci_data_analysis/analysis/matrix_helper.py b/cdci_data_analysis/analysis/matrix_helper.py index b6d98d56c..8ef865278 100644 --- a/cdci_data_analysis/analysis/matrix_helper.py +++ b/cdci_data_analysis/analysis/matrix_helper.py @@ -188,7 +188,7 @@ def send_job_message( matrix_sender_access_token = config.matrix_sender_access_token receiver_room_id = tokenHelper.get_token_user_matrix_room_id(decoded_token) - cc_receivers_room_id = config.matrix_server_url + cc_receivers_room_id = config.matrix_cc_receivers_room_id matrix_message_data = { 'oda_site': { From 8c18423dd349a9f1ca6faa11f72e18a6e6d2b909 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 12 Oct 2023 15:28:03 +0200 Subject: [PATCH 05/23] ne sentry if parameter is missing --- cdci_data_analysis/analysis/matrix_helper.py | 17 +++++++++++------ .../flask_app/dispatcher_query.py | 17 ++++++++--------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/cdci_data_analysis/analysis/matrix_helper.py b/cdci_data_analysis/analysis/matrix_helper.py index 8ef865278..a3b90f0d8 100644 --- a/cdci_data_analysis/analysis/matrix_helper.py +++ b/cdci_data_analysis/analysis/matrix_helper.py @@ -258,12 +258,6 @@ def send_message( if sender_access_token is None: matrix_helper_logger.info('matrix sender_access_token not available') raise MissingRequestParameter('matrix sender_access_token not available') - if room_id is None: - matrix_helper_logger.info('matrix room_id not available') - raise MissingRequestParameter('matrix room_id not available') - if message_text is None or message_body_html is None: - matrix_helper_logger.info('matrix message not available') - raise MissingRequestParameter('matrix message not available') matrix_helper_logger.info(f"Sending message to the room id: {room_id}") url = os.path.join(url_server, f'_matrix/client/r0/rooms/{room_id}/send/m.room.message') @@ -400,6 +394,17 @@ def is_message_to_send_run_query(status, time_original_request, scratch_dir, job return sending_ok +def is_matrix_config_present(config): + url_server = config.matrix_server_url + sender_access_token = config.matrix_sender_access_token + + if url_server is None or sender_access_token is None: + matrix_helper_logger.info('matrix url server not available') + return False + + return True + + def is_message_to_send_callback(status, time_original_request, scratch_dir, config, job_id, decoded_token=None): log_additional_info_obj = {} sending_ok = False diff --git a/cdci_data_analysis/flask_app/dispatcher_query.py b/cdci_data_analysis/flask_app/dispatcher_query.py index d498041e7..f95caaff6 100644 --- a/cdci_data_analysis/flask_app/dispatcher_query.py +++ b/cdci_data_analysis/flask_app/dispatcher_query.py @@ -1045,13 +1045,13 @@ def run_call_back(self, status_kw_name='action') -> typing.Tuple[str, typing.Uni is_email_to_send = False is_message_to_send = False try: - step = 'checking if an email can be sent' + step = 'checking if a message can be sent via matrix' is_message_to_send = matrix_helper.is_message_to_send_callback(status, - time_original_request, - self.scratch_dir, - self.app.config['conf'], - self.job_id, - decoded_token=self.decoded_token) + time_original_request, + self.scratch_dir, + self.app.config['conf'], + self.job_id, + decoded_token=self.decoded_token) except matrix_helper.MultipleDoneMatrixMessage as e: job.write_dataserver_status(status_dictionary_value=status, full_dict=self.par_dic, @@ -1060,7 +1060,7 @@ def run_call_back(self, status_kw_name='action') -> typing.Tuple[str, typing.Uni sentry.capture_message(f'attempted repeated sending of completion matrix message detected: {e}') try: - step = 'checking if a message can be sent via matrix' + step = 'checking if an email can be sent' is_email_to_send = email_helper.is_email_to_send_callback(self.logger, status, time_original_request, @@ -1971,8 +1971,7 @@ def run_query(self, off_line=False, disp_conf=None): except MissingRequestParameter as e: query_out.set_status_field('matrix_message_status', 'sending matrix message failed') query_out.set_status_field('matrix_message_status_details', e.payload) - logging.warning(f'matrix message sending failed: {e}') - sentry.capture_message(f'sending matrix message failed {e.message}') + logging.warning(f'parameter missing when sending a message via matrix: {e.message}') if email_helper.is_email_to_send_run_query(self.logger, query_new_status, From 626adfa8f609621277f76e14d006eb8105d8379d Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 12 Oct 2023 15:39:43 +0200 Subject: [PATCH 06/23] including data messages sent to token user and cc users --- cdci_data_analysis/analysis/matrix_helper.py | 32 ++++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/cdci_data_analysis/analysis/matrix_helper.py b/cdci_data_analysis/analysis/matrix_helper.py index a3b90f0d8..8ed38683f 100644 --- a/cdci_data_analysis/analysis/matrix_helper.py +++ b/cdci_data_analysis/analysis/matrix_helper.py @@ -220,23 +220,37 @@ def send_job_message( template = env.get_template('matrix_message.html') message_body_html = template.render(**matrix_message_data) message_text = textify_matrix_message(message_body_html) - res_data = send_message(url_server=matrix_server_url, - sender_access_token=matrix_sender_access_token, - room_id=receiver_room_id, - message_text=message_text, - message_body_html=message_body_html - ) + res_data_message_token_user = send_message(url_server=matrix_server_url, + sender_access_token=matrix_sender_access_token, + room_id=receiver_room_id, + message_text=message_text, + message_body_html=message_body_html + ) + message_data_token_user = res_data_message_token_user['message_data'] + res_content_token_user = res_data_message_token_user['res_content'] + + res_content = { + 'res_content_token_user': res_content_token_user, + 'res_content_cc_users': [] + } + + message_data = { + 'message_data_token_user': message_data_token_user, + 'message_data_cc_users': [] + } for cc_room_id in cc_receivers_room_id: - res_data = send_message(url_server=matrix_server_url, + res_data_message_cc_user = send_message(url_server=matrix_server_url, sender_access_token=matrix_sender_access_token, room_id=cc_room_id, message_text=message_text, message_body_html=message_body_html ) + message_data_cc_user = res_data_message_cc_user['message_data'] + message_data['message_data_cc_users'].append(message_data_cc_user) + res_content_cc_user = res_data_message_cc_user['res_content'] + res_content['res_content_cc_users'].append(res_content_cc_user) - message_data = res_data['message_data'] - res_content = res_data['res_content'] store_status_matrix_message_info(message_data, status, scratch_dir, sending_time=sending_time, first_submitted_time=time_request) From 62ba92c2cd76c74e01c565b929d07109094f471f Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 12 Oct 2023 15:55:54 +0200 Subject: [PATCH 07/23] log message if no user room id within the token --- cdci_data_analysis/analysis/matrix_helper.py | 69 ++++++++++++-------- 1 file changed, 42 insertions(+), 27 deletions(-) diff --git a/cdci_data_analysis/analysis/matrix_helper.py b/cdci_data_analysis/analysis/matrix_helper.py index 8ed38683f..f10b29aab 100644 --- a/cdci_data_analysis/analysis/matrix_helper.py +++ b/cdci_data_analysis/analysis/matrix_helper.py @@ -220,32 +220,35 @@ def send_job_message( template = env.get_template('matrix_message.html') message_body_html = template.render(**matrix_message_data) message_text = textify_matrix_message(message_body_html) - res_data_message_token_user = send_message(url_server=matrix_server_url, - sender_access_token=matrix_sender_access_token, - room_id=receiver_room_id, - message_text=message_text, - message_body_html=message_body_html - ) - message_data_token_user = res_data_message_token_user['message_data'] - res_content_token_user = res_data_message_token_user['res_content'] - res_content = { - 'res_content_token_user': res_content_token_user, 'res_content_cc_users': [] } message_data = { - 'message_data_token_user': message_data_token_user, 'message_data_cc_users': [] } + if receiver_room_id is not None: + res_data_message_token_user = send_message(url_server=matrix_server_url, + sender_access_token=matrix_sender_access_token, + room_id=receiver_room_id, + message_text=message_text, + message_body_html=message_body_html + ) + message_data_token_user = res_data_message_token_user['message_data'] + res_content_token_user = res_data_message_token_user['res_content'] + message_data['message_data_token_user'] = message_data_token_user + res_content['res_content_token_user'] = res_content_token_user + else: + matrix_helper_logger.warning('a matrix message could not be sent to the token user as no personal room id was ' + 'provided within the token') - for cc_room_id in cc_receivers_room_id: + for cc_receiver_room_id in cc_receivers_room_id: res_data_message_cc_user = send_message(url_server=matrix_server_url, - sender_access_token=matrix_sender_access_token, - room_id=cc_room_id, - message_text=message_text, - message_body_html=message_body_html - ) + sender_access_token=matrix_sender_access_token, + room_id=cc_receiver_room_id, + message_text=message_text, + message_body_html=message_body_html + ) message_data_cc_user = res_data_message_cc_user['message_data'] message_data['message_data_cc_users'].append(message_data_cc_user) res_content_cc_user = res_data_message_cc_user['res_content'] @@ -265,14 +268,6 @@ def send_message( message_text=None, message_body_html=None, ): - - if url_server is None: - matrix_helper_logger.info('matrix url server not available') - raise MissingRequestParameter('matrix url server not available') - if sender_access_token is None: - matrix_helper_logger.info('matrix sender_access_token not available') - raise MissingRequestParameter('matrix sender_access_token not available') - matrix_helper_logger.info(f"Sending message to the room id: {room_id}") url = os.path.join(url_server, f'_matrix/client/r0/rooms/{room_id}/send/m.room.message') @@ -318,12 +313,31 @@ def send_message( return res_data +def is_matrix_config_ok(config): + if config.matrix_server_url is None: + matrix_helper_logger.info('matrix url server not available') + return False + if config.matrix_sender_access_token is None: + matrix_helper_logger.info('matrix sender_access_token not available') + return False + return True + + def is_message_to_send_run_query(status, time_original_request, scratch_dir, job_id, config, decoded_token=None): log_additional_info_obj = {} sending_ok = False + config_ok = is_matrix_config_ok(config) time_check = time_.time() sentry_for_matrix_message_sending_check = config.sentry_for_matrix_message_sending_check + + if config.matrix_server_url is None: + matrix_helper_logger.info('matrix url server not available') + config_ok = False + if config.matrix_sender_access_token is None: + matrix_helper_logger.info('matrix sender_access_token not available') + config_ok = False + # get total request duration if decoded_token: # in case the job is just submitted and was not submitted before, at least since some time @@ -405,7 +419,7 @@ def is_message_to_send_run_query(status, time_original_request, scratch_dir, job else: matrix_helper_logger.info(f'a message on matrix will not be sent because a token was not provided') - return sending_ok + return sending_ok and config_ok def is_matrix_config_present(config): @@ -422,6 +436,7 @@ def is_matrix_config_present(config): def is_message_to_send_callback(status, time_original_request, scratch_dir, config, job_id, decoded_token=None): log_additional_info_obj = {} sending_ok = False + config_ok = is_matrix_config_ok(config) time_check = time_.time() sentry_for_matrix_message_sending_check = config.sentry_for_matrix_message_sending_check @@ -496,7 +511,7 @@ def is_message_to_send_callback(status, time_original_request, scratch_dir, conf additional_info_obj=log_additional_info_obj ) - return sending_ok + return sending_ok and config_ok def log_matrix_message_sending_info(status, time_request, scratch_dir, job_id, additional_info_obj=None): From 0d2d8f59373e8694f3869041a1749cf49034b2d4 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 12 Oct 2023 15:59:32 +0200 Subject: [PATCH 08/23] except MissingRequestParameter when checking is message/email can be sent, no sentry, run_call_back --- .../flask_app/dispatcher_query.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/cdci_data_analysis/flask_app/dispatcher_query.py b/cdci_data_analysis/flask_app/dispatcher_query.py index f95caaff6..79ddb1537 100644 --- a/cdci_data_analysis/flask_app/dispatcher_query.py +++ b/cdci_data_analysis/flask_app/dispatcher_query.py @@ -1058,6 +1058,11 @@ def run_call_back(self, status_kw_name='action') -> typing.Tuple[str, typing.Uni matrix_message_status='attempted repeated sending of matrix message detected') logging.warning(f'attempted repeated sending of completion matrix message detected: {e}') sentry.capture_message(f'attempted repeated sending of completion matrix message detected: {e}') + except MissingRequestParameter as e: + job.write_dataserver_status(status_dictionary_value=status, + full_dict=self.par_dic, + call_back_status=f'parameter missing when checking if a message via matrix could be sent: {e.message}') + logging.warning(f'parameter missing when checking if a message via matrix could be sent: {e.message}') try: step = 'checking if an email can be sent' @@ -1074,6 +1079,12 @@ def run_call_back(self, status_kw_name='action') -> typing.Tuple[str, typing.Uni email_status='attempted repeated sending of completion email detected') logging.warning(f'attempted repeated sending of completion email detected: {e}') sentry.capture_message(f'attempted repeated sending of completion email detected: {e}') + except MissingRequestParameter as e: + job.write_dataserver_status(status_dictionary_value=status, + full_dict=self.par_dic, + call_back_status=f'parameter missing when checking if an could be sent: {e.message}') + logging.warning(f'parameter missing when checking if an could be sent: {e.message}') + try: if is_email_to_send or is_message_to_send: step = 'extracting the original request dictionary' @@ -1151,12 +1162,6 @@ def run_call_back(self, status_kw_name='action') -> typing.Tuple[str, typing.Uni logging.warning(f'matrix message sending failed: {e}') sentry.capture_message(f'sending matrix message failed {e.message}') - except MissingRequestParameter as e: - job.write_dataserver_status(status_dictionary_value=status, - full_dict=self.par_dic, - call_back_status=f'parameter missing when sending a message via matrix: {e.message}') - logging.warning(f'parameter missing when sending a message via matrix: {e.message}') - try: # TODO for a future implementation # self.validate_job_id() @@ -1199,11 +1204,6 @@ def run_call_back(self, status_kw_name='action') -> typing.Tuple[str, typing.Uni logging.warning(f'email sending failed: {e}') sentry.capture_message(f'sending email failed {e}') - except MissingRequestParameter as e: - job.write_dataserver_status(status_dictionary_value=status, - full_dict=self.par_dic, - call_back_status=f'parameter missing when sending an email: {e.message}') - logging.warning(f'parameter missing when sending an email: {e.message}') # TODO for a future implementation # except RequestNotAuthorized as e: # job.write_dataserver_status(status_dictionary_value=status, From 904cbb8e78b12c1e9469d2864f86144e03127e96 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 12 Oct 2023 16:02:09 +0200 Subject: [PATCH 09/23] no catching MissingRequestParameter when sending job message run_query --- cdci_data_analysis/flask_app/dispatcher_query.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cdci_data_analysis/flask_app/dispatcher_query.py b/cdci_data_analysis/flask_app/dispatcher_query.py index 79ddb1537..a571e938d 100644 --- a/cdci_data_analysis/flask_app/dispatcher_query.py +++ b/cdci_data_analysis/flask_app/dispatcher_query.py @@ -1968,10 +1968,6 @@ def run_query(self, off_line=False, disp_conf=None): query_out.set_status_field('matrix_message_status_details', e.payload) logging.warning(f'matrix message sending failed: {e}') sentry.capture_message(f'sending matrix message failed {e.message}') - except MissingRequestParameter as e: - query_out.set_status_field('matrix_message_status', 'sending matrix message failed') - query_out.set_status_field('matrix_message_status_details', e.payload) - logging.warning(f'parameter missing when sending a message via matrix: {e.message}') if email_helper.is_email_to_send_run_query(self.logger, query_new_status, From f3ccf3a39b795c232475cca3328de4ba866f9316 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 12 Oct 2023 16:24:54 +0200 Subject: [PATCH 10/23] testing adaption --- tests/test_matrix_messages.py | 41 +++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 12 deletions(-) diff --git a/tests/test_matrix_messages.py b/tests/test_matrix_messages.py index 4226e2cd4..31d303998 100644 --- a/tests/test_matrix_messages.py +++ b/tests/test_matrix_messages.py @@ -284,13 +284,18 @@ def test_matrix_message_run_analysis_callback(gunicorn_dispatcher_long_living_fi # matrix message not supposed to be sent for public request assert 'matrix_message_status' not in jdata else: + assert 'matrix_message_status' in jdata['exit_status'] assert jdata['exit_status']['matrix_message_status'] == 'matrix message sent' assert 'matrix_message_status_details' in jdata['exit_status'] - matrix_message_event_id_obj = json.loads(jdata['exit_status']['matrix_message_status_details']) - assert 'event_id' in matrix_message_event_id_obj['res_content'] + matrix_message_status_details_obj = json.loads(jdata['exit_status']['matrix_message_status_details']) + assert 'res_content' in matrix_message_status_details_obj + assert 'res_content_cc_users' in matrix_message_status_details_obj['res_content'] + assert matrix_message_status_details_obj['res_content']['res_content_cc_users'] == [] + assert 'res_content_token_user' in matrix_message_status_details_obj['res_content'] + assert 'event_id' in matrix_message_status_details_obj['res_content']['res_content_token_user'] - matrix_message_event_id_obj = matrix_message_event_id_obj['res_content']['event_id'] + matrix_message_event_id_obj = matrix_message_status_details_obj['res_content']['res_content_token_user']['event_id'] validate_matrix_message_content( dispatcher_local_matrix_message_server.get_matrix_message_record(room_id=token_payload['mxroomid'], @@ -428,10 +433,14 @@ def test_matrix_message_run_analysis_callback(gunicorn_dispatcher_long_living_fi assert 'matrix_message_status' in jdata assert jdata['matrix_message_status'] == 'matrix message sent' assert 'matrix_message_status_details' in jdata - matrix_message_event_id_obj = json.loads(jdata['matrix_message_status_details']) - assert 'event_id' in matrix_message_event_id_obj['res_content'] - - matrix_message_event_id_obj = matrix_message_event_id_obj['res_content']['event_id'] + matrix_message_status_details_obj = json.loads(jdata['matrix_message_status_details']) + assert 'res_content' in matrix_message_status_details_obj + assert 'res_content_cc_users' in matrix_message_status_details_obj['res_content'] + assert matrix_message_status_details_obj['res_content']['res_content_cc_users'] == [] + assert 'res_content_token_user' in matrix_message_status_details_obj['res_content'] + assert 'event_id' in matrix_message_status_details_obj['res_content']['res_content_token_user'] + + matrix_message_event_id_obj = matrix_message_status_details_obj['res_content']['res_content_token_user']['event_id'] # check the matrix message in the matrix message folders, and that the first one was produced dispatcher_job_state.assert_matrix_message(state="done") @@ -473,10 +482,14 @@ def test_matrix_message_run_analysis_callback(gunicorn_dispatcher_long_living_fi assert 'matrix_message_status' in jdata assert jdata['matrix_message_status'] == 'matrix message sent' assert 'matrix_message_status_details' in jdata - matrix_message_event_id_obj = json.loads(jdata['matrix_message_status_details']) - assert 'event_id' in matrix_message_event_id_obj['res_content'] + matrix_message_status_details_obj = json.loads(jdata['matrix_message_status_details']) + assert 'res_content' in matrix_message_status_details_obj + assert 'res_content_cc_users' in matrix_message_status_details_obj['res_content'] + assert matrix_message_status_details_obj['res_content']['res_content_cc_users'] == [] + assert 'res_content_token_user' in matrix_message_status_details_obj['res_content'] + assert 'event_id' in matrix_message_status_details_obj['res_content']['res_content_token_user'] - matrix_message_event_id_obj = matrix_message_event_id_obj['res_content']['event_id'] + matrix_message_event_id_obj = matrix_message_status_details_obj['res_content']['res_content_token_user']['event_id'] # check the matrix message in the matrix message folders, and that the first one was produced if default_values or time_original_request_none: @@ -542,8 +555,12 @@ def test_matrix_message_submitted_same_job(dispatcher_live_fixture_with_matrix_o assert 'matrix_message_status' in jdata['exit_status'] assert jdata['exit_status']['matrix_message_status'] == 'matrix message sent' assert 'matrix_message_status_details' in jdata['exit_status'] - matrix_message_event_id_obj = json.loads(jdata['exit_status']['matrix_message_status_details']) - assert 'event_id' in matrix_message_event_id_obj['res_content'] + matrix_message_status_details_obj = json.loads(jdata['exit_status']['matrix_message_status_details']) + assert 'res_content' in matrix_message_status_details_obj + assert 'res_content_cc_users' in matrix_message_status_details_obj['res_content'] + assert matrix_message_status_details_obj['res_content']['res_content_cc_users'] == [] + assert 'res_content_token_user' in matrix_message_status_details_obj['res_content'] + assert 'event_id' in matrix_message_status_details_obj['res_content']['res_content_token_user'] time_request = jdata['time_request'] time_request_str = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(float(time_request))) From 5101b223bb6386a0ce4d8dd801cad4eccdfb8f2e Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 12 Oct 2023 16:33:45 +0200 Subject: [PATCH 11/23] testing adaption --- tests/test_matrix_messages.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_matrix_messages.py b/tests/test_matrix_messages.py index 31d303998..5078f4cf1 100644 --- a/tests/test_matrix_messages.py +++ b/tests/test_matrix_messages.py @@ -970,15 +970,15 @@ def test_status_details_matrix_message_done(gunicorn_dispatcher_long_living_fixt job_id=dispatcher_job_state.job_id, token=encoded_token) - matrix_message_event_id_obj = matrix_message_event_id_obj['res_content']['event_id'] + matrix_message_event_id = matrix_message_event_id_obj['res_content']['res_content_token_user']['event_id'] # check the email in the log files validate_matrix_message_content( dispatcher_local_matrix_message_server.get_matrix_message_record(room_id=token_payload['mxroomid'], - event_id=matrix_message_event_id_obj), + event_id=matrix_message_event_id), 'done', room_id=token_payload['mxroomid'], - event_id=matrix_message_event_id_obj, + event_id=matrix_message_event_id, user_id=token_payload['user_id'], dispatcher_job_state=dispatcher_job_state, variation_suffixes=["failing"], From 91de612353bf4f84b4d3c864a43aa0b93cb68eb7 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 12 Oct 2023 22:02:25 +0200 Subject: [PATCH 12/23] sending incident report --- cdci_data_analysis/analysis/matrix_helper.py | 40 ++++++++++++++++---- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/cdci_data_analysis/analysis/matrix_helper.py b/cdci_data_analysis/analysis/matrix_helper.py index f10b29aab..347a52a4c 100644 --- a/cdci_data_analysis/analysis/matrix_helper.py +++ b/cdci_data_analysis/analysis/matrix_helper.py @@ -106,6 +106,7 @@ def send_incident_report_message( matrix_server_url = config.matrix_server_url matrix_sender_access_token = config.matrix_sender_access_token receiver_room_id = tokenHelper.get_token_user_matrix_room_id(decoded_token) + cc_receivers_room_id = config.matrix_cc_receivers_room_id matrix_message_data = { 'request': { @@ -126,16 +127,39 @@ def send_incident_report_message( # open("debug_email_lines_too_long.html", "w").write(email_body_html) # open("debug_email_lines_too_long.text", "w").write(email_text) # raise MatrixMessageNotSent(f"message not sent on matrix, lines too long!") + res_content = { + 'res_content_cc_users': [] + } - res_data = send_message(url_server=matrix_server_url, - sender_access_token=matrix_sender_access_token, - room_id=receiver_room_id, - message_text=message_text, - message_body_html=message_body_html - ) + message_data = { + 'message_data_cc_users': [] + } + if receiver_room_id is not None: + res_data_message_token_user = send_message(url_server=matrix_server_url, + sender_access_token=matrix_sender_access_token, + room_id=receiver_room_id, + message_text=message_text, + message_body_html=message_body_html + ) + message_data_token_user = res_data_message_token_user['message_data'] + res_content_token_user = res_data_message_token_user['res_content'] + message_data['message_data_token_user'] = message_data_token_user + res_content['res_content_token_user'] = res_content_token_user + else: + matrix_helper_logger.warning('a matrix message could not be sent to the token user as no personal room id was ' + 'provided within the token') - message_data = res_data['message_data'] - res_content = res_data['res_content'] + for cc_receiver_room_id in cc_receivers_room_id: + res_data_message_cc_user = send_message(url_server=matrix_server_url, + sender_access_token=matrix_sender_access_token, + room_id=cc_receiver_room_id, + message_text=message_text, + message_body_html=message_body_html + ) + message_data_cc_user = res_data_message_cc_user['message_data'] + message_data['message_data_cc_users'].append(message_data_cc_user) + res_content_cc_user = res_data_message_cc_user['res_content'] + res_content['res_content_cc_users'].append(res_content_cc_user) store_incident_report_matrix_message(message_data, scratch_dir, sending_time=sending_time) From 545c9e687242bd8e41bc5029c961a1b6391c1e29 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 12 Oct 2023 23:18:49 +0200 Subject: [PATCH 13/23] matrix_incident_report_receivers_room_ids and matrix_incident_report_sender_personal_access_token configs --- cdci_data_analysis/configurer.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cdci_data_analysis/configurer.py b/cdci_data_analysis/configurer.py index 0fa8ce179..511e011af 100644 --- a/cdci_data_analysis/configurer.py +++ b/cdci_data_analysis/configurer.py @@ -248,6 +248,8 @@ def __init__(self, cfg_dict, origin=None): disp_dict.get('matrix_options', {}).get('matrix_server_url', None), disp_dict.get('matrix_options', {}).get('matrix_sender_access_token', None), disp_dict.get('matrix_options', {}).get('matrix_cc_receivers_room_id', []), + disp_dict.get('matrix_options', {}).get('matrix_incident_report_receivers_room_ids', []), + disp_dict.get('matrix_options', {}).get('matrix_incident_report_sender_personal_access_token', None), disp_dict.get('matrix_options', {}).get('matrix_message_sending_job_submitted', True), disp_dict.get('matrix_options', {}).get('matrix_message_sending_job_submitted_default_interval', 1800), disp_dict.get('matrix_options', {}).get('sentry_for_matrix_message_sending_check', False), @@ -324,6 +326,8 @@ def set_conf_dispatcher(self, matrix_server_url, matrix_sender_access_token, matrix_cc_receivers_room_id, + matrix_incident_report_receivers_room_ids, + matrix_incident_report_sender_personal_access_token, matrix_message_sending_job_submitted, matrix_message_sending_job_submitted_default_interval, sentry_for_matrix_message_sending_check, @@ -372,6 +376,8 @@ def set_conf_dispatcher(self, self.matrix_server_url = matrix_server_url self.matrix_sender_access_token = matrix_sender_access_token self.matrix_cc_receivers_room_id = matrix_cc_receivers_room_id + self.matrix_incident_report_receivers_room_ids = matrix_incident_report_receivers_room_ids + self.matrix_incident_report_sender_personal_access_token = matrix_incident_report_sender_personal_access_token self.matrix_message_sending_job_submitted = matrix_message_sending_job_submitted self.matrix_message_sending_job_submitted_default_interval = matrix_message_sending_job_submitted_default_interval self.sentry_for_matrix_message_sending_check = sentry_for_matrix_message_sending_check From 9d1ae7dbbc809cc9c517e0d0d0e7dcf3ebc57776 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 12 Oct 2023 23:19:00 +0200 Subject: [PATCH 14/23] matrix_incident_report_receivers_room_ids and matrix_incident_report_sender_personal_access_token test values --- cdci_data_analysis/pytest_fixtures.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cdci_data_analysis/pytest_fixtures.py b/cdci_data_analysis/pytest_fixtures.py index fa5f35faf..fd1261fe5 100644 --- a/cdci_data_analysis/pytest_fixtures.py +++ b/cdci_data_analysis/pytest_fixtures.py @@ -564,7 +564,9 @@ def dispatcher_test_conf_with_matrix_options_fn(dispatcher_test_conf_fn): f.write('\n matrix_options:' '\n matrix_server_url: "https://matrix-client.matrix.org/"' f'\n matrix_sender_access_token: "{os.getenv("MATRIX_SENDER_ACCESS_TOKEN", "matrix_sender_access_token")}"' - '\n matrix_cc_receivers_room_id: []' + f'\n matrix_cc_receivers_room_id: ["{os.getenv("MATRIX_CC_RECEIVER_ROOM_ID", "matrix_cc_receivers_room_id")}"]' + f'\n matrix_incident_report_receivers_room_ids: ["{os.getenv("MATRIX_INCIDENT_REPORT_RECEIVER_ROOM_ID", "matrix_incident_report_receivers_room_id")}"]' + f'\n matrix_incident_report_sender_personal_access_token: "{os.getenv("MATRIX_INCIDENT_REPORT_SENDER_PERSONAL_ACCESS_TOKEN", "matrix_incident_report_sender_personal_access_token")}"' '\n matrix_message_sending_job_submitted: True' '\n matrix_message_sending_job_submitted_default_interval: 5' '\n sentry_for_matrix_message_sending_check: False' From 894a7d94dec292812596cecb62c4a97f0f981c9c Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 12 Oct 2023 23:19:47 +0200 Subject: [PATCH 15/23] sending report using config --- cdci_data_analysis/analysis/matrix_helper.py | 38 ++++++-------------- 1 file changed, 11 insertions(+), 27 deletions(-) diff --git a/cdci_data_analysis/analysis/matrix_helper.py b/cdci_data_analysis/analysis/matrix_helper.py index 347a52a4c..fbb949be6 100644 --- a/cdci_data_analysis/analysis/matrix_helper.py +++ b/cdci_data_analysis/analysis/matrix_helper.py @@ -104,9 +104,9 @@ def send_incident_report_message( env.filters['humanize_future'] = humanize_future matrix_server_url = config.matrix_server_url - matrix_sender_access_token = config.matrix_sender_access_token - receiver_room_id = tokenHelper.get_token_user_matrix_room_id(decoded_token) - cc_receivers_room_id = config.matrix_cc_receivers_room_id + + incident_report_receivers_room_ids = config.matrix_incident_report_receivers_room_ids + incident_report_sender_personal_access_token = config.matrix_incident_report_sender_personal_access_token matrix_message_data = { 'request': { @@ -128,38 +128,22 @@ def send_incident_report_message( # open("debug_email_lines_too_long.text", "w").write(email_text) # raise MatrixMessageNotSent(f"message not sent on matrix, lines too long!") res_content = { - 'res_content_cc_users': [] + 'res_content_incident_reports': [] } message_data = { - 'message_data_cc_users': [] + 'message_data_incident_reports': [] } - if receiver_room_id is not None: - res_data_message_token_user = send_message(url_server=matrix_server_url, - sender_access_token=matrix_sender_access_token, - room_id=receiver_room_id, - message_text=message_text, - message_body_html=message_body_html - ) - message_data_token_user = res_data_message_token_user['message_data'] - res_content_token_user = res_data_message_token_user['res_content'] - message_data['message_data_token_user'] = message_data_token_user - res_content['res_content_token_user'] = res_content_token_user - else: - matrix_helper_logger.warning('a matrix message could not be sent to the token user as no personal room id was ' - 'provided within the token') - for cc_receiver_room_id in cc_receivers_room_id: - res_data_message_cc_user = send_message(url_server=matrix_server_url, - sender_access_token=matrix_sender_access_token, - room_id=cc_receiver_room_id, + for incident_report_receiver_room_id in incident_report_receivers_room_ids: + res_data_message_receiver = send_message(url_server=matrix_server_url, + sender_access_token=incident_report_sender_personal_access_token, + room_id=incident_report_receiver_room_id, message_text=message_text, message_body_html=message_body_html ) - message_data_cc_user = res_data_message_cc_user['message_data'] - message_data['message_data_cc_users'].append(message_data_cc_user) - res_content_cc_user = res_data_message_cc_user['res_content'] - res_content['res_content_cc_users'].append(res_content_cc_user) + message_data['message_data_incident_reports'].append(res_data_message_receiver['message_data']) + res_content['res_content_incident_reports'].append(res_data_message_receiver['res_content']) store_incident_report_matrix_message(message_data, scratch_dir, sending_time=sending_time) From 52fb0dcd07ed3e3162378f8f3f000206bfcdf1f9 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 12 Oct 2023 23:19:52 +0200 Subject: [PATCH 16/23] extended test --- tests/test_matrix_messages.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/test_matrix_messages.py b/tests/test_matrix_messages.py index 5078f4cf1..9f2708e2b 100644 --- a/tests/test_matrix_messages.py +++ b/tests/test_matrix_messages.py @@ -102,7 +102,6 @@ def validate_incident_matrix_message_content( message_record, room_id:str, event_id:str, - user_id:str, dispatcher_job_state: DispatcherJobState, incident_time_str: str = None, incident_report_str: str = None, @@ -121,7 +120,6 @@ def validate_incident_matrix_message_content( ) assert message_record['room_id'] == room_id - assert message_record['user_id'] == user_id assert message_record['type'] == 'm.room.message' assert message_record['event_id'] == event_id @@ -1061,6 +1059,7 @@ def test_matrix_message_and_email(gunicorn_dispatcher_long_living_fixture_with_m @pytest.mark.test_matrix @pytest.mark.parametrize("request_cred", ['public', 'valid_token', 'invalid_token']) def test_incident_report(dispatcher_live_fixture_with_matrix_options, + dispatcher_test_conf_with_matrix_options, dispatcher_local_matrix_message_server, dispatcher_test_conf, request_cred): @@ -1129,15 +1128,16 @@ def test_incident_report(dispatcher_live_fixture_with_matrix_options, assert jdata_incident_report['martix_message_report_status'] == 'incident report message successfully sent via matrix' assert 'martix_message_report_status_details' in jdata_incident_report assert 'res_content' in jdata_incident_report['martix_message_report_status_details'] - assert 'event_id' in jdata_incident_report['martix_message_report_status_details']['res_content'] - matrix_message_event_id = jdata_incident_report['martix_message_report_status_details']['res_content']['event_id'] + assert 'res_content_incident_reports' in jdata_incident_report['martix_message_report_status_details']['res_content'] + assert len(jdata_incident_report['martix_message_report_status_details']['res_content']['res_content_incident_reports']) == 1 + assert 'event_id' in jdata_incident_report['martix_message_report_status_details']['res_content']['res_content_incident_reports'][0] + matrix_message_event_id = jdata_incident_report['martix_message_report_status_details']['res_content']['res_content_incident_reports'][0]['event_id'] validate_incident_matrix_message_content( - dispatcher_local_matrix_message_server.get_matrix_message_record(room_id=decoded_token['mxroomid'], + dispatcher_local_matrix_message_server.get_matrix_message_record(room_id=dispatcher_test_conf_with_matrix_options['matrix_options']['matrix_incident_report_receivers_room_ids'][0], event_id=matrix_message_event_id), event_id=matrix_message_event_id, - room_id=decoded_token['mxroomid'], - user_id=decoded_token['user_id'], + room_id=dispatcher_test_conf_with_matrix_options['matrix_options']['matrix_incident_report_receivers_room_ids'][0], dispatcher_job_state=dispatcher_job_state, incident_time_str=time_request_str, incident_report_str=incident_content, From 33595b4f3df47a0236224f9da724bc4b31aa8367 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Thu, 12 Oct 2023 23:44:31 +0200 Subject: [PATCH 17/23] check receiver room id is not empty and not None, extended test --- cdci_data_analysis/analysis/matrix_helper.py | 32 ++++++++++---------- cdci_data_analysis/pytest_fixtures.py | 2 +- tests/test_matrix_messages.py | 6 ++-- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/cdci_data_analysis/analysis/matrix_helper.py b/cdci_data_analysis/analysis/matrix_helper.py index fbb949be6..c2936e997 100644 --- a/cdci_data_analysis/analysis/matrix_helper.py +++ b/cdci_data_analysis/analysis/matrix_helper.py @@ -235,7 +235,7 @@ def send_job_message( message_data = { 'message_data_cc_users': [] } - if receiver_room_id is not None: + if receiver_room_id is not None and receiver_room_id != "": res_data_message_token_user = send_message(url_server=matrix_server_url, sender_access_token=matrix_sender_access_token, room_id=receiver_room_id, @@ -251,16 +251,17 @@ def send_job_message( 'provided within the token') for cc_receiver_room_id in cc_receivers_room_id: - res_data_message_cc_user = send_message(url_server=matrix_server_url, - sender_access_token=matrix_sender_access_token, - room_id=cc_receiver_room_id, - message_text=message_text, - message_body_html=message_body_html - ) - message_data_cc_user = res_data_message_cc_user['message_data'] - message_data['message_data_cc_users'].append(message_data_cc_user) - res_content_cc_user = res_data_message_cc_user['res_content'] - res_content['res_content_cc_users'].append(res_content_cc_user) + if cc_receiver_room_id is not None and cc_receiver_room_id != "": + res_data_message_cc_user = send_message(url_server=matrix_server_url, + sender_access_token=matrix_sender_access_token, + room_id=cc_receiver_room_id, + message_text=message_text, + message_body_html=message_body_html + ) + message_data_cc_user = res_data_message_cc_user['message_data'] + message_data['message_data_cc_users'].append(message_data_cc_user) + res_content_cc_user = res_data_message_cc_user['res_content'] + res_content['res_content_cc_users'].append(res_content_cc_user) store_status_matrix_message_info(message_data, status, scratch_dir, sending_time=sending_time, first_submitted_time=time_request) @@ -268,7 +269,6 @@ def send_job_message( return res_content - def send_message( url_server=None, sender_access_token=None, @@ -300,14 +300,14 @@ def send_message( except json.decoder.JSONDecodeError: error_msg = res.text matrix_helper_logger.warning(f"there seems to be some problem in sending a message via matrix:\n" - f"the requested url {url} lead to the error {error_msg}, " - "this might be due to an error in the url or the page requested no longer exists, " - "please check it and try to issue again the request") + f"the requested url {url} lead to the error {error_msg}, " + "this might be due to an error in the url or the page requested no longer exists, " + "please check it and try to issue again the request") matrix_error_message = error_msg sentry.capture_message(f'issue in sending a message via matrix, the requested url {url} lead to the error ' f'{matrix_error_message}') - raise MatrixMessageNotSent('issue when performing a request to the product gallery', + raise MatrixMessageNotSent('issue in sending a message via matrix', status_code=res.status_code, payload={'matrix_error_message': matrix_error_message}) diff --git a/cdci_data_analysis/pytest_fixtures.py b/cdci_data_analysis/pytest_fixtures.py index fd1261fe5..fe7bb8882 100644 --- a/cdci_data_analysis/pytest_fixtures.py +++ b/cdci_data_analysis/pytest_fixtures.py @@ -564,7 +564,7 @@ def dispatcher_test_conf_with_matrix_options_fn(dispatcher_test_conf_fn): f.write('\n matrix_options:' '\n matrix_server_url: "https://matrix-client.matrix.org/"' f'\n matrix_sender_access_token: "{os.getenv("MATRIX_SENDER_ACCESS_TOKEN", "matrix_sender_access_token")}"' - f'\n matrix_cc_receivers_room_id: ["{os.getenv("MATRIX_CC_RECEIVER_ROOM_ID", "matrix_cc_receivers_room_id")}"]' + f'\n matrix_cc_receivers_room_id: ["{os.getenv("MATRIX_CC_RECEIVER_ROOM_ID", "")}"]' f'\n matrix_incident_report_receivers_room_ids: ["{os.getenv("MATRIX_INCIDENT_REPORT_RECEIVER_ROOM_ID", "matrix_incident_report_receivers_room_id")}"]' f'\n matrix_incident_report_sender_personal_access_token: "{os.getenv("MATRIX_INCIDENT_REPORT_SENDER_PERSONAL_ACCESS_TOKEN", "matrix_incident_report_sender_personal_access_token")}"' '\n matrix_message_sending_job_submitted: True' diff --git a/tests/test_matrix_messages.py b/tests/test_matrix_messages.py index 9f2708e2b..d6dd7ea16 100644 --- a/tests/test_matrix_messages.py +++ b/tests/test_matrix_messages.py @@ -555,10 +555,12 @@ def test_matrix_message_submitted_same_job(dispatcher_live_fixture_with_matrix_o assert 'matrix_message_status_details' in jdata['exit_status'] matrix_message_status_details_obj = json.loads(jdata['exit_status']['matrix_message_status_details']) assert 'res_content' in matrix_message_status_details_obj - assert 'res_content_cc_users' in matrix_message_status_details_obj['res_content'] - assert matrix_message_status_details_obj['res_content']['res_content_cc_users'] == [] assert 'res_content_token_user' in matrix_message_status_details_obj['res_content'] assert 'event_id' in matrix_message_status_details_obj['res_content']['res_content_token_user'] + assert 'res_content_cc_users' in matrix_message_status_details_obj['res_content'] + assert isinstance(matrix_message_status_details_obj['res_content']['res_content_cc_users'], list) + assert len(matrix_message_status_details_obj['res_content']['res_content_cc_users']) == 1 + assert 'event_id' in matrix_message_status_details_obj['res_content']['res_content_cc_users'][0] time_request = jdata['time_request'] time_request_str = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(float(time_request))) From c6e0c62732ffbd3205cbb2d8b6169da82b33a2cd Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 13 Oct 2023 10:21:12 +0200 Subject: [PATCH 18/23] config renaming --- cdci_data_analysis/analysis/matrix_helper.py | 4 ++-- cdci_data_analysis/config_dir/conf_env.yml.example | 8 ++++++++ cdci_data_analysis/configurer.py | 10 +++++----- cdci_data_analysis/pytest_fixtures.py | 7 ++++--- 4 files changed, 19 insertions(+), 10 deletions(-) diff --git a/cdci_data_analysis/analysis/matrix_helper.py b/cdci_data_analysis/analysis/matrix_helper.py index c2936e997..dd2250d05 100644 --- a/cdci_data_analysis/analysis/matrix_helper.py +++ b/cdci_data_analysis/analysis/matrix_helper.py @@ -196,7 +196,7 @@ def send_job_message( matrix_sender_access_token = config.matrix_sender_access_token receiver_room_id = tokenHelper.get_token_user_matrix_room_id(decoded_token) - cc_receivers_room_id = config.matrix_cc_receivers_room_id + cc_receivers_room_ids = config.matrix_cc_receivers_room_ids matrix_message_data = { 'oda_site': { @@ -250,7 +250,7 @@ def send_job_message( matrix_helper_logger.warning('a matrix message could not be sent to the token user as no personal room id was ' 'provided within the token') - for cc_receiver_room_id in cc_receivers_room_id: + for cc_receiver_room_id in cc_receivers_room_ids: if cc_receiver_room_id is not None and cc_receiver_room_id != "": res_data_message_cc_user = send_message(url_server=matrix_server_url, sender_access_token=matrix_sender_access_token, diff --git a/cdci_data_analysis/config_dir/conf_env.yml.example b/cdci_data_analysis/config_dir/conf_env.yml.example index 65846701d..f69fddd93 100644 --- a/cdci_data_analysis/config_dir/conf_env.yml.example +++ b/cdci_data_analysis/config_dir/conf_env.yml.example @@ -80,6 +80,14 @@ dispatcher: matrix_server_url: MATRIX_SERVER_URL # access token of the sender account, from which the messages will be sent from matrix_sender_access_token: MATRIX_SENDER_ACCESS_TOKEN + # list of additional room ids receivers towards which the message will be sent, besides the room id extracted from the token + matrix_cc_receivers_room_ids: ['room_id'] + # incident report related options, for the messages sent via matrix + incident_report_matrix_options: + # list of room ids receivers towards which the incident report message will be sent + matrix_incident_report_receivers_room_ids: ['room_id'] + # personal access token of the sender of the incident report + matrix_incident_report_sender_personal_access_token: MATRIX_INCIDENT_REPORT_SENDER_ACCESS_TOKEN # list of room ids to which the matrix message should be sent matrix_cc_receivers_room_id: [] # enable/disable sending of messages via matrix in case of a submitted job status diff --git a/cdci_data_analysis/configurer.py b/cdci_data_analysis/configurer.py index 511e011af..f862b23a4 100644 --- a/cdci_data_analysis/configurer.py +++ b/cdci_data_analysis/configurer.py @@ -247,9 +247,9 @@ def __init__(self, cfg_dict, origin=None): disp_dict['email_options'].get('incident_report_email_options', {}).get('incident_report_receivers_email_addresses', None), disp_dict.get('matrix_options', {}).get('matrix_server_url', None), disp_dict.get('matrix_options', {}).get('matrix_sender_access_token', None), - disp_dict.get('matrix_options', {}).get('matrix_cc_receivers_room_id', []), - disp_dict.get('matrix_options', {}).get('matrix_incident_report_receivers_room_ids', []), - disp_dict.get('matrix_options', {}).get('matrix_incident_report_sender_personal_access_token', None), + disp_dict.get('matrix_options', {}).get('matrix_cc_receivers_room_ids', []), + disp_dict.get('matrix_options', {}).get('incident_report_matrix_options', {}).get('matrix_incident_report_receivers_room_ids', []), + disp_dict.get('matrix_options', {}).get('incident_report_matrix_options', {}).get('matrix_incident_report_sender_personal_access_token', None), disp_dict.get('matrix_options', {}).get('matrix_message_sending_job_submitted', True), disp_dict.get('matrix_options', {}).get('matrix_message_sending_job_submitted_default_interval', 1800), disp_dict.get('matrix_options', {}).get('sentry_for_matrix_message_sending_check', False), @@ -325,7 +325,7 @@ def set_conf_dispatcher(self, incident_report_receivers_email_addresses, matrix_server_url, matrix_sender_access_token, - matrix_cc_receivers_room_id, + matrix_cc_receivers_room_ids, matrix_incident_report_receivers_room_ids, matrix_incident_report_sender_personal_access_token, matrix_message_sending_job_submitted, @@ -375,7 +375,7 @@ def set_conf_dispatcher(self, self.incident_report_receivers_email_addresses = incident_report_receivers_email_addresses self.matrix_server_url = matrix_server_url self.matrix_sender_access_token = matrix_sender_access_token - self.matrix_cc_receivers_room_id = matrix_cc_receivers_room_id + self.matrix_cc_receivers_room_ids = matrix_cc_receivers_room_ids self.matrix_incident_report_receivers_room_ids = matrix_incident_report_receivers_room_ids self.matrix_incident_report_sender_personal_access_token = matrix_incident_report_sender_personal_access_token self.matrix_message_sending_job_submitted = matrix_message_sending_job_submitted diff --git a/cdci_data_analysis/pytest_fixtures.py b/cdci_data_analysis/pytest_fixtures.py index fe7bb8882..59d0bfbb8 100644 --- a/cdci_data_analysis/pytest_fixtures.py +++ b/cdci_data_analysis/pytest_fixtures.py @@ -564,9 +564,10 @@ def dispatcher_test_conf_with_matrix_options_fn(dispatcher_test_conf_fn): f.write('\n matrix_options:' '\n matrix_server_url: "https://matrix-client.matrix.org/"' f'\n matrix_sender_access_token: "{os.getenv("MATRIX_SENDER_ACCESS_TOKEN", "matrix_sender_access_token")}"' - f'\n matrix_cc_receivers_room_id: ["{os.getenv("MATRIX_CC_RECEIVER_ROOM_ID", "")}"]' - f'\n matrix_incident_report_receivers_room_ids: ["{os.getenv("MATRIX_INCIDENT_REPORT_RECEIVER_ROOM_ID", "matrix_incident_report_receivers_room_id")}"]' - f'\n matrix_incident_report_sender_personal_access_token: "{os.getenv("MATRIX_INCIDENT_REPORT_SENDER_PERSONAL_ACCESS_TOKEN", "matrix_incident_report_sender_personal_access_token")}"' + f'\n matrix_cc_receivers_room_ids: ["{os.getenv("MATRIX_CC_RECEIVER_ROOM_ID", "")}"]' + '\n incident_report_matrix_options:' + f'\n matrix_incident_report_receivers_room_ids: ["{os.getenv("MATRIX_INCIDENT_REPORT_RECEIVER_ROOM_ID", "matrix_incident_report_receivers_room_ids")}"]' + f'\n matrix_incident_report_sender_personal_access_token: "{os.getenv("MATRIX_INCIDENT_REPORT_SENDER_PERSONAL_ACCESS_TOKEN", "matrix_incident_report_sender_personal_access_token")}"' '\n matrix_message_sending_job_submitted: True' '\n matrix_message_sending_job_submitted_default_interval: 5' '\n sentry_for_matrix_message_sending_check: False' From 91cc8a7cb2892c829c9a12bd2433d89ae1968df4 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 13 Oct 2023 10:21:20 +0200 Subject: [PATCH 19/23] tests adapted --- tests/test_matrix_messages.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/test_matrix_messages.py b/tests/test_matrix_messages.py index d6dd7ea16..c53440288 100644 --- a/tests/test_matrix_messages.py +++ b/tests/test_matrix_messages.py @@ -289,7 +289,8 @@ def test_matrix_message_run_analysis_callback(gunicorn_dispatcher_long_living_fi matrix_message_status_details_obj = json.loads(jdata['exit_status']['matrix_message_status_details']) assert 'res_content' in matrix_message_status_details_obj assert 'res_content_cc_users' in matrix_message_status_details_obj['res_content'] - assert matrix_message_status_details_obj['res_content']['res_content_cc_users'] == [] + assert isinstance(matrix_message_status_details_obj['res_content']['res_content_cc_users'], list) + assert len(matrix_message_status_details_obj['res_content']['res_content_cc_users']) == 1 assert 'res_content_token_user' in matrix_message_status_details_obj['res_content'] assert 'event_id' in matrix_message_status_details_obj['res_content']['res_content_token_user'] @@ -434,7 +435,8 @@ def test_matrix_message_run_analysis_callback(gunicorn_dispatcher_long_living_fi matrix_message_status_details_obj = json.loads(jdata['matrix_message_status_details']) assert 'res_content' in matrix_message_status_details_obj assert 'res_content_cc_users' in matrix_message_status_details_obj['res_content'] - assert matrix_message_status_details_obj['res_content']['res_content_cc_users'] == [] + assert isinstance(matrix_message_status_details_obj['res_content']['res_content_cc_users'], list) + assert len(matrix_message_status_details_obj['res_content']['res_content_cc_users']) == 1 assert 'res_content_token_user' in matrix_message_status_details_obj['res_content'] assert 'event_id' in matrix_message_status_details_obj['res_content']['res_content_token_user'] @@ -483,7 +485,8 @@ def test_matrix_message_run_analysis_callback(gunicorn_dispatcher_long_living_fi matrix_message_status_details_obj = json.loads(jdata['matrix_message_status_details']) assert 'res_content' in matrix_message_status_details_obj assert 'res_content_cc_users' in matrix_message_status_details_obj['res_content'] - assert matrix_message_status_details_obj['res_content']['res_content_cc_users'] == [] + assert isinstance(matrix_message_status_details_obj['res_content']['res_content_cc_users'], list) + assert len(matrix_message_status_details_obj['res_content']['res_content_cc_users']) == 1 assert 'res_content_token_user' in matrix_message_status_details_obj['res_content'] assert 'event_id' in matrix_message_status_details_obj['res_content']['res_content_token_user'] @@ -1136,10 +1139,10 @@ def test_incident_report(dispatcher_live_fixture_with_matrix_options, matrix_message_event_id = jdata_incident_report['martix_message_report_status_details']['res_content']['res_content_incident_reports'][0]['event_id'] validate_incident_matrix_message_content( - dispatcher_local_matrix_message_server.get_matrix_message_record(room_id=dispatcher_test_conf_with_matrix_options['matrix_options']['matrix_incident_report_receivers_room_ids'][0], + dispatcher_local_matrix_message_server.get_matrix_message_record(room_id=dispatcher_test_conf_with_matrix_options['matrix_options']['incident_report_matrix_options']['matrix_incident_report_receivers_room_ids'][0], event_id=matrix_message_event_id), event_id=matrix_message_event_id, - room_id=dispatcher_test_conf_with_matrix_options['matrix_options']['matrix_incident_report_receivers_room_ids'][0], + room_id=dispatcher_test_conf_with_matrix_options['matrix_options']['incident_report_matrix_options']['matrix_incident_report_receivers_room_ids'][0], dispatcher_job_state=dispatcher_job_state, incident_time_str=time_request_str, incident_report_str=incident_content, From e2173a7abf7a7ebaf808df2f972ef628e63bf4ac Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 13 Oct 2023 10:58:28 +0200 Subject: [PATCH 20/23] logging rephrasing --- cdci_data_analysis/flask_app/dispatcher_query.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cdci_data_analysis/flask_app/dispatcher_query.py b/cdci_data_analysis/flask_app/dispatcher_query.py index a571e938d..0a879f6f4 100644 --- a/cdci_data_analysis/flask_app/dispatcher_query.py +++ b/cdci_data_analysis/flask_app/dispatcher_query.py @@ -1061,8 +1061,8 @@ def run_call_back(self, status_kw_name='action') -> typing.Tuple[str, typing.Uni except MissingRequestParameter as e: job.write_dataserver_status(status_dictionary_value=status, full_dict=self.par_dic, - call_back_status=f'parameter missing when checking if a message via matrix could be sent: {e.message}') - logging.warning(f'parameter missing when checking if a message via matrix could be sent: {e.message}') + call_back_status=f'parameter missing when checking if a message could be sent via matrix: {e.message}') + logging.warning(f'parameter missing when checking if a message could be sent via matrix: {e.message}') try: step = 'checking if an email can be sent' @@ -1082,8 +1082,8 @@ def run_call_back(self, status_kw_name='action') -> typing.Tuple[str, typing.Uni except MissingRequestParameter as e: job.write_dataserver_status(status_dictionary_value=status, full_dict=self.par_dic, - call_back_status=f'parameter missing when checking if an could be sent: {e.message}') - logging.warning(f'parameter missing when checking if an could be sent: {e.message}') + call_back_status=f'parameter missing when checking if an email could be sent: {e.message}') + logging.warning(f'parameter missing when checking if an email could be sent: {e.message}') try: if is_email_to_send or is_message_to_send: From 73a3bcec5f8163cfd6a424971d609fa6a3bb998e Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 13 Oct 2023 11:42:40 +0200 Subject: [PATCH 21/23] includes case no room id in the token --- tests/test_matrix_messages.py | 130 ++++++++++++++++++++++++---------- 1 file changed, 92 insertions(+), 38 deletions(-) diff --git a/tests/test_matrix_messages.py b/tests/test_matrix_messages.py index c53440288..9626893f1 100644 --- a/tests/test_matrix_messages.py +++ b/tests/test_matrix_messages.py @@ -190,8 +190,9 @@ def adapt_html(html_content, patterns=None, **matrix_message_args,): @pytest.mark.test_matrix @pytest.mark.parametrize("default_values", [True, False]) @pytest.mark.parametrize("time_original_request_none", [False]) -@pytest.mark.parametrize("request_cred", ['public', 'private', 'private-no-matrix-message']) +@pytest.mark.parametrize("request_cred", ['public', 'private', 'private-no-matrix-message', 'private-no-room-id']) def test_matrix_message_run_analysis_callback(gunicorn_dispatcher_long_living_fixture_with_matrix_options, + dispatcher_test_conf_with_matrix_options, dispatcher_local_matrix_message_server, default_values, request_cred, time_original_request_none): DataServerQuery.set_status('submitted') @@ -231,6 +232,8 @@ def test_matrix_message_run_analysis_callback(gunicorn_dispatcher_long_living_fi token_payload['mxdone'] = False token_payload['mxfail'] = False expect_matrix_message = False + elif request_cred == 'private-no-room-id': + token_payload.pop('mxroomid', None) encoded_token = jwt.encode(token_payload, secret_key, algorithm='HS256') @@ -288,20 +291,43 @@ def test_matrix_message_run_analysis_callback(gunicorn_dispatcher_long_living_fi assert 'matrix_message_status_details' in jdata['exit_status'] matrix_message_status_details_obj = json.loads(jdata['exit_status']['matrix_message_status_details']) assert 'res_content' in matrix_message_status_details_obj + if request_cred == 'private-no-room-id': + assert 'res_content_token_user' not in matrix_message_status_details_obj['res_content'] + else: + assert 'res_content_token_user' in matrix_message_status_details_obj['res_content'] + matrix_user_message_event_id = \ + matrix_message_status_details_obj['res_content']['res_content_token_user']['event_id'] + + validate_matrix_message_content( + dispatcher_local_matrix_message_server.get_matrix_message_record( + room_id=token_payload['mxroomid'], + event_id=matrix_user_message_event_id), + 'submitted', + room_id=token_payload['mxroomid'], + event_id=matrix_user_message_event_id, + user_id=token_payload['user_id'], + dispatcher_job_state=dispatcher_job_state, + variation_suffixes=["dummy"], + time_request_str=time_request_str, + request_params=dict_param, + products_url=products_url, + dispatcher_live_fixture=None, + require_reference_matrix_message=True + ) + assert 'res_content_cc_users' in matrix_message_status_details_obj['res_content'] assert isinstance(matrix_message_status_details_obj['res_content']['res_content_cc_users'], list) assert len(matrix_message_status_details_obj['res_content']['res_content_cc_users']) == 1 - assert 'res_content_token_user' in matrix_message_status_details_obj['res_content'] - assert 'event_id' in matrix_message_status_details_obj['res_content']['res_content_token_user'] + assert 'event_id' in matrix_message_status_details_obj['res_content']['res_content_cc_users'][0] - matrix_message_event_id_obj = matrix_message_status_details_obj['res_content']['res_content_token_user']['event_id'] + matrix_cc_message_event_id_obj = matrix_message_status_details_obj['res_content']['res_content_cc_users'][0]['event_id'] validate_matrix_message_content( - dispatcher_local_matrix_message_server.get_matrix_message_record(room_id=token_payload['mxroomid'], - event_id=matrix_message_event_id_obj), + dispatcher_local_matrix_message_server.get_matrix_message_record(room_id=dispatcher_test_conf_with_matrix_options['matrix_options']['matrix_cc_receivers_room_ids'][0], + event_id=matrix_cc_message_event_id_obj), 'submitted', - room_id=token_payload['mxroomid'], - event_id=matrix_message_event_id_obj, + room_id=dispatcher_test_conf_with_matrix_options['matrix_options']['matrix_cc_receivers_room_ids'][0], + event_id=matrix_cc_message_event_id_obj, user_id=token_payload['user_id'], dispatcher_job_state=dispatcher_job_state, variation_suffixes=["dummy"], @@ -437,19 +463,42 @@ def test_matrix_message_run_analysis_callback(gunicorn_dispatcher_long_living_fi assert 'res_content_cc_users' in matrix_message_status_details_obj['res_content'] assert isinstance(matrix_message_status_details_obj['res_content']['res_content_cc_users'], list) assert len(matrix_message_status_details_obj['res_content']['res_content_cc_users']) == 1 - assert 'res_content_token_user' in matrix_message_status_details_obj['res_content'] - assert 'event_id' in matrix_message_status_details_obj['res_content']['res_content_token_user'] - matrix_message_event_id_obj = matrix_message_status_details_obj['res_content']['res_content_token_user']['event_id'] + if request_cred == 'private-no-room-id': + assert 'res_content_token_user' not in matrix_message_status_details_obj['res_content'] + else: + assert 'res_content_token_user' in matrix_message_status_details_obj['res_content'] + assert 'event_id' in matrix_message_status_details_obj['res_content']['res_content_token_user'] + matrix_user_message_event_id = \ + matrix_message_status_details_obj['res_content']['res_content_token_user']['event_id'] + + validate_matrix_message_content( + dispatcher_local_matrix_message_server.get_matrix_message_record( + room_id=token_payload['mxroomid'], + event_id=matrix_user_message_event_id), + 'done', + room_id=token_payload['mxroomid'], + event_id=matrix_user_message_event_id, + user_id=token_payload['user_id'], + dispatcher_job_state=dispatcher_job_state, + time_request_str=time_request_str, + dispatcher_live_fixture=server, + require_reference_matrix_message=True + ) + + matrix_cc_message_event_id_obj = matrix_message_status_details_obj['res_content']['res_content_cc_users'][0][ + 'event_id'] + # check the matrix message in the matrix message folders, and that the first one was produced dispatcher_job_state.assert_matrix_message(state="done") validate_matrix_message_content( - dispatcher_local_matrix_message_server.get_matrix_message_record(room_id=token_payload['mxroomid'], - event_id=matrix_message_event_id_obj), + dispatcher_local_matrix_message_server.get_matrix_message_record( + room_id=dispatcher_test_conf_with_matrix_options['matrix_options']['matrix_cc_receivers_room_ids'][0], + event_id=matrix_cc_message_event_id_obj), 'done', - room_id=token_payload['mxroomid'], - event_id=matrix_message_event_id_obj, + room_id=dispatcher_test_conf_with_matrix_options['matrix_options']['matrix_cc_receivers_room_ids'][0], + event_id=matrix_cc_message_event_id_obj, user_id=token_payload['user_id'], dispatcher_job_state=dispatcher_job_state, time_request_str=time_request_str, @@ -487,29 +536,32 @@ def test_matrix_message_run_analysis_callback(gunicorn_dispatcher_long_living_fi assert 'res_content_cc_users' in matrix_message_status_details_obj['res_content'] assert isinstance(matrix_message_status_details_obj['res_content']['res_content_cc_users'], list) assert len(matrix_message_status_details_obj['res_content']['res_content_cc_users']) == 1 - assert 'res_content_token_user' in matrix_message_status_details_obj['res_content'] - assert 'event_id' in matrix_message_status_details_obj['res_content']['res_content_token_user'] - - matrix_message_event_id_obj = matrix_message_status_details_obj['res_content']['res_content_token_user']['event_id'] - - # check the matrix message in the matrix message folders, and that the first one was produced - if default_values or time_original_request_none: - dispatcher_job_state.assert_matrix_message('failed', comment="expected one matrix message in total, failed") + if request_cred == 'private-no-room-id': + assert 'res_content_token_user' not in matrix_message_status_details_obj['res_content'] else: - dispatcher_job_state.assert_matrix_message('failed', comment="expected two matrix message in total, second failed") - - validate_matrix_message_content( - dispatcher_local_matrix_message_server.get_matrix_message_record(room_id=token_payload['mxroomid'], - event_id=matrix_message_event_id_obj), - 'failed', - room_id=token_payload['mxroomid'], - event_id=matrix_message_event_id_obj, - user_id=token_payload['user_id'], - dispatcher_job_state=dispatcher_job_state, - time_request_str=time_request_str, - dispatcher_live_fixture=server, - require_reference_matrix_message=True - ) + assert 'res_content_token_user' in matrix_message_status_details_obj['res_content'] + assert 'event_id' in matrix_message_status_details_obj['res_content']['res_content_token_user'] + + matrix_message_event_id_obj = matrix_message_status_details_obj['res_content']['res_content_token_user']['event_id'] + + # check the matrix message in the matrix message folders, and that the first one was produced + if default_values or time_original_request_none: + dispatcher_job_state.assert_matrix_message('failed', comment="expected one matrix message in total, failed") + else: + dispatcher_job_state.assert_matrix_message('failed', comment="expected two matrix message in total, second failed") + + validate_matrix_message_content( + dispatcher_local_matrix_message_server.get_matrix_message_record(room_id=token_payload['mxroomid'], + event_id=matrix_message_event_id_obj), + 'failed', + room_id=token_payload['mxroomid'], + event_id=matrix_message_event_id_obj, + user_id=token_payload['user_id'], + dispatcher_job_state=dispatcher_job_state, + time_request_str=time_request_str, + dispatcher_live_fixture=server, + require_reference_matrix_message=True + ) @pytest.mark.test_matrix @@ -1147,4 +1199,6 @@ def test_incident_report(dispatcher_live_fixture_with_matrix_options, incident_time_str=time_request_str, incident_report_str=incident_content, decoded_token=decoded_token - ) \ No newline at end of file + ) + + From b417e83013d4d294bb92328282a8776897922f6c Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 13 Oct 2023 11:52:32 +0200 Subject: [PATCH 22/23] test no cc room_ids --- cdci_data_analysis/pytest_fixtures.py | 5 + tests/conftest.py | 1 + tests/test_matrix_messages.py | 340 ++++++++++++++++++++++++++ 3 files changed, 346 insertions(+) diff --git a/cdci_data_analysis/pytest_fixtures.py b/cdci_data_analysis/pytest_fixtures.py index 59d0bfbb8..9a5d26a90 100644 --- a/cdci_data_analysis/pytest_fixtures.py +++ b/cdci_data_analysis/pytest_fixtures.py @@ -577,6 +577,11 @@ def dispatcher_test_conf_with_matrix_options_fn(dispatcher_test_conf_fn): yield fn +@pytest.fixture +def dispatcher_no_cc_matrix_room_ids(monkeypatch): + monkeypatch.delenv('MATRIX_CC_RECEIVER_ROOM_ID', raising=False) + + @pytest.fixture def dispatcher_test_conf_with_gallery_no_resolver_fn(dispatcher_test_conf_fn): fn = "test-dispatcher-conf-with-gallery.yaml" diff --git a/tests/conftest.py b/tests/conftest.py index be7962151..846829e46 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,6 +14,7 @@ dispatcher_long_living_fixture, gunicorn_dispatcher_long_living_fixture, dispatcher_long_living_fixture_with_matrix_options, + dispatcher_no_cc_matrix_room_ids, gunicorn_dispatcher_long_living_fixture_with_matrix_options, dispatcher_test_conf, dispatcher_test_conf_with_gallery, diff --git a/tests/test_matrix_messages.py b/tests/test_matrix_messages.py index 9626893f1..20d64fcdd 100644 --- a/tests/test_matrix_messages.py +++ b/tests/test_matrix_messages.py @@ -1202,3 +1202,343 @@ def test_incident_report(dispatcher_live_fixture_with_matrix_options, ) +@pytest.mark.test_matrix +@pytest.mark.parametrize("default_values", [True, False]) +@pytest.mark.parametrize("time_original_request_none", [False]) +@pytest.mark.parametrize("request_cred", ['public', 'private', 'private-no-matrix-message', 'private-no-room-id']) +def test_matrix_message_run_analysis_callback_no_room_ids(dispatcher_no_cc_matrix_room_ids, + gunicorn_dispatcher_long_living_fixture_with_matrix_options, + dispatcher_test_conf_with_matrix_options, + dispatcher_local_matrix_message_server, + default_values, request_cred, time_original_request_none): + DataServerQuery.set_status('submitted') + + server = gunicorn_dispatcher_long_living_fixture_with_matrix_options + + DispatcherJobState.remove_scratch_folders() + + token_none = (request_cred == 'public') + + expect_matrix_message = True + token_payload = {**default_token_payload, + "tmx": 0, + "tem": 0, + "mxstout": True, + "mxintsub": 5, + "mxsub": True, + "mssub": False, + "msdone": False, + "msfail": False, + "mxroomid": dispatcher_local_matrix_message_server.room_id + } + + if token_none: + encoded_token = None + else: + # let's generate a valid token with high threshold + + if default_values: + token_payload.pop('tmx') + token_payload.pop('mxstout') + token_payload.pop('mxsub') + token_payload.pop('mxintsub') + + if request_cred == 'private-no-matrix-message': + token_payload['mxsub'] = False + token_payload['mxdone'] = False + token_payload['mxfail'] = False + expect_matrix_message = False + elif request_cred == 'private-no-room-id': + token_payload.pop('mxroomid', None) + + encoded_token = jwt.encode(token_payload, secret_key, algorithm='HS256') + + dict_param = dict( + query_status="new", + query_type="Real", + instrument="empty-async", + product_type="dummy", + token=encoded_token + ) + + # this should return status submitted, so matrix message sent + c = requests.get(os.path.join(server, "run_analysis"), + dict_param + ) + assert c.status_code == 200 + jdata = c.json() + + session_id = jdata['session_id'] + job_id = jdata['job_monitor']['job_id'] + + logger.info("response from run_analysis: %s", json.dumps(jdata, indent=4)) + dispatcher_job_state = DispatcherJobState.from_run_analysis_response(c.json()) + + assert jdata['query_status'] == "submitted" + + completed_dict_param = {**dict_param, + 'use_scws': 'no', + 'src_name': '1E 1740.7-2942', + 'RA': 265.97845833, + 'DEC': -29.74516667, + 'T1': '2017-03-06T13:26:48.000', + 'T2': '2017-03-06T15:32:27.000', + 'T_format': 'isot' + } + + products_url = DispatcherJobState.get_expected_products_url(completed_dict_param, + token=encoded_token, + session_id=session_id, + job_id=job_id) + assert jdata['exit_status']['job_status'] == 'submitted' + # get the original time the request was made + assert 'time_request' in jdata + # set the time the request was initiated + time_request = jdata['time_request'] + time_request_str = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(float(time_request))) + + if token_none or not expect_matrix_message: + # matrix message not supposed to be sent for public request + assert 'matrix_message_status' not in jdata + else: + + assert 'matrix_message_status' in jdata['exit_status'] + assert jdata['exit_status']['matrix_message_status'] == 'matrix message sent' + assert 'matrix_message_status_details' in jdata['exit_status'] + matrix_message_status_details_obj = json.loads(jdata['exit_status']['matrix_message_status_details']) + assert 'res_content' in matrix_message_status_details_obj + if request_cred == 'private-no-room-id': + assert 'res_content_token_user' not in matrix_message_status_details_obj['res_content'] + else: + assert 'res_content_token_user' in matrix_message_status_details_obj['res_content'] + matrix_user_message_event_id = \ + matrix_message_status_details_obj['res_content']['res_content_token_user']['event_id'] + + validate_matrix_message_content( + dispatcher_local_matrix_message_server.get_matrix_message_record( + room_id=token_payload['mxroomid'], + event_id=matrix_user_message_event_id), + 'submitted', + room_id=token_payload['mxroomid'], + event_id=matrix_user_message_event_id, + user_id=token_payload['user_id'], + dispatcher_job_state=dispatcher_job_state, + variation_suffixes=["dummy"], + time_request_str=time_request_str, + request_params=dict_param, + products_url=products_url, + dispatcher_live_fixture=None, + require_reference_matrix_message=True + ) + + assert 'res_content_cc_users' in matrix_message_status_details_obj['res_content'] + assert isinstance(matrix_message_status_details_obj['res_content']['res_content_cc_users'], list) + assert len(matrix_message_status_details_obj['res_content']['res_content_cc_users']) == 0 + + # for the call_back(s) in case the time of the original request is not provided + if time_original_request_none: + time_request = None + time_request_str = 'None' + + for i in range(5): + # imitating what a backend would do + current_action = 'progress' if i > 2 else 'main_done' + c = requests.get(os.path.join(server, "call_back"), + params=dict( + job_id=dispatcher_job_state.job_id, + session_id=dispatcher_job_state.session_id, + instrument_name="empty-async", + action=current_action, + node_id=f'node_{i}', + message='progressing', + token=encoded_token, + time_original_request=time_request + )) + assert dispatcher_job_state.load_job_state_record(f'node_{i}', "progressing")['full_report_dict'][ + 'action'] == current_action + + c = requests.get(os.path.join(server, "run_analysis"), + params=dict( + query_status="submitted", # whether query is new or not, this should work + query_type="Real", + instrument="empty-async", + product_type="dummy", + async_dispatcher=False, + session_id=dispatcher_job_state.session_id, + job_id=dispatcher_job_state.job_id, + token=encoded_token + )) + assert c.json()['query_status'] == 'progress' # always progress! + + # we should now find progress records + c = requests.get(os.path.join(server, "run_analysis"), + {**dict_param, + "query_status": "submitted", + "job_id": job_id, + "session_id": session_id, + } + ) + + assert c.status_code == 200 + jdata = c.json() + + assert len(jdata['job_monitor']['full_report_dict_list']) == 6 + assert [c['action'] for c in jdata['job_monitor']['full_report_dict_list']] == [ + 'main_done', 'main_done', 'main_done', 'progress', 'progress', 'progress'] + + c = requests.get(os.path.join(server, "call_back"), + params=dict( + job_id=dispatcher_job_state.job_id, + session_id=dispatcher_job_state.session_id, + instrument_name="empty-async", + action='main_incorrect_status', + node_id=f'node_{i + 1}', + message='progressing', + token=encoded_token, + time_original_request=time_request + )) + assert c.status_code == 200 + + c = requests.get(os.path.join(server, "run_analysis"), + { + **dict_param, + "query_status": "submitted", + "job_id": job_id, + "session_id": session_id, + } + ) + assert c.status_code == 200 + assert c.json()['query_status'] == 'progress' + + # this does nothing special + c = requests.get(os.path.join(server, "call_back"), + params=dict( + job_id=dispatcher_job_state.job_id, + session_id=dispatcher_job_state.session_id, + instrument_name="empty-async", + action='ready', + node_id='node_ready', + message='ready', + token=encoded_token, + time_original_request=time_request + )) + + DataServerQuery.set_status('done') + + # this triggers a message via matrix + c = requests.get(os.path.join(server, "call_back"), + params=dict( + job_id=dispatcher_job_state.job_id, + session_id=dispatcher_job_state.session_id, + instrument_name="empty-async", + action='done', + node_id='node_final', + message='done', + token=encoded_token, + time_original_request=time_request + )) + + assert c.status_code == 200 + + jdata = dispatcher_job_state.load_job_state_record('node_final', 'done') + + if token_none or not expect_matrix_message: + assert 'matrix_message_status' not in jdata + + elif time_original_request_none: + assert 'matrix_message_status' in jdata + + elif default_values: + assert 'matrix_message_status' not in jdata + + else: + assert 'matrix_message_status' in jdata + assert jdata['matrix_message_status'] == 'matrix message sent' + assert 'matrix_message_status_details' in jdata + matrix_message_status_details_obj = json.loads(jdata['matrix_message_status_details']) + assert 'res_content' in matrix_message_status_details_obj + assert 'res_content_cc_users' in matrix_message_status_details_obj['res_content'] + assert isinstance(matrix_message_status_details_obj['res_content']['res_content_cc_users'], list) + assert len(matrix_message_status_details_obj['res_content']['res_content_cc_users']) == 0 + + if request_cred == 'private-no-room-id': + assert 'res_content_token_user' not in matrix_message_status_details_obj['res_content'] + else: + assert 'res_content_token_user' in matrix_message_status_details_obj['res_content'] + assert 'event_id' in matrix_message_status_details_obj['res_content']['res_content_token_user'] + matrix_user_message_event_id = \ + matrix_message_status_details_obj['res_content']['res_content_token_user']['event_id'] + + validate_matrix_message_content( + dispatcher_local_matrix_message_server.get_matrix_message_record( + room_id=token_payload['mxroomid'], + event_id=matrix_user_message_event_id), + 'done', + room_id=token_payload['mxroomid'], + event_id=matrix_user_message_event_id, + user_id=token_payload['user_id'], + dispatcher_job_state=dispatcher_job_state, + time_request_str=time_request_str, + dispatcher_live_fixture=server, + require_reference_matrix_message=True + ) + + # check the matrix message in the matrix message folders, and that the first one was produced + dispatcher_job_state.assert_matrix_message(state="done") + + # this also triggers matrix message (simulate a failed request) + c = requests.get(os.path.join(server, "call_back"), + params={ + 'job_id': dispatcher_job_state.job_id, + 'session_id': dispatcher_job_state.session_id, + 'instrument_name': "empty-async", + 'action': 'failed', + 'node_id': 'node_failed', + 'message': 'failed', + 'token': encoded_token, + 'time_original_request': time_request + }) + + assert c.status_code == 200 + + jdata = dispatcher_job_state.load_job_state_record('node_failed', 'failed') + + if token_none or not expect_matrix_message: + # matrix message not supposed to be sent for public request + assert 'matrix_message_status' not in jdata + assert 'matrix_message_status_details' not in jdata + else: + assert 'matrix_message_status' in jdata + assert jdata['matrix_message_status'] == 'matrix message sent' + assert 'matrix_message_status_details' in jdata + matrix_message_status_details_obj = json.loads(jdata['matrix_message_status_details']) + assert 'res_content' in matrix_message_status_details_obj + assert 'res_content_cc_users' in matrix_message_status_details_obj['res_content'] + assert isinstance(matrix_message_status_details_obj['res_content']['res_content_cc_users'], list) + assert len(matrix_message_status_details_obj['res_content']['res_content_cc_users']) == 0 + if request_cred == 'private-no-room-id': + assert 'res_content_token_user' not in matrix_message_status_details_obj['res_content'] + else: + assert 'res_content_token_user' in matrix_message_status_details_obj['res_content'] + assert 'event_id' in matrix_message_status_details_obj['res_content']['res_content_token_user'] + + matrix_message_event_id_obj = matrix_message_status_details_obj['res_content']['res_content_token_user']['event_id'] + + # check the matrix message in the matrix message folders, and that the first one was produced + if default_values or time_original_request_none: + dispatcher_job_state.assert_matrix_message('failed', comment="expected one matrix message in total, failed") + else: + dispatcher_job_state.assert_matrix_message('failed', comment="expected two matrix message in total, second failed") + + validate_matrix_message_content( + dispatcher_local_matrix_message_server.get_matrix_message_record(room_id=token_payload['mxroomid'], + event_id=matrix_message_event_id_obj), + 'failed', + room_id=token_payload['mxroomid'], + event_id=matrix_message_event_id_obj, + user_id=token_payload['user_id'], + dispatcher_job_state=dispatcher_job_state, + time_request_str=time_request_str, + dispatcher_live_fixture=server, + require_reference_matrix_message=True + ) From 30906e94da4c9a947bbda8bb2671759df20416a8 Mon Sep 17 00:00:00 2001 From: burnout87 Date: Fri, 13 Oct 2023 12:07:42 +0200 Subject: [PATCH 23/23] matrix_cc_receivers_room_id to matrix_bcc_receivers_room_id --- cdci_data_analysis/analysis/matrix_helper.py | 16 ++--- .../config_dir/conf_env.yml.example | 2 +- cdci_data_analysis/configurer.py | 6 +- cdci_data_analysis/pytest_fixtures.py | 4 +- tests/conftest.py | 2 +- tests/test_matrix_messages.py | 68 +++++++++---------- 6 files changed, 49 insertions(+), 49 deletions(-) diff --git a/cdci_data_analysis/analysis/matrix_helper.py b/cdci_data_analysis/analysis/matrix_helper.py index dd2250d05..70992025a 100644 --- a/cdci_data_analysis/analysis/matrix_helper.py +++ b/cdci_data_analysis/analysis/matrix_helper.py @@ -196,7 +196,7 @@ def send_job_message( matrix_sender_access_token = config.matrix_sender_access_token receiver_room_id = tokenHelper.get_token_user_matrix_room_id(decoded_token) - cc_receivers_room_ids = config.matrix_cc_receivers_room_ids + bcc_receivers_room_ids = config.matrix_bcc_receivers_room_ids matrix_message_data = { 'oda_site': { @@ -229,11 +229,11 @@ def send_job_message( message_body_html = template.render(**matrix_message_data) message_text = textify_matrix_message(message_body_html) res_content = { - 'res_content_cc_users': [] + 'res_content_bcc_users': [] } message_data = { - 'message_data_cc_users': [] + 'message_data_bcc_users': [] } if receiver_room_id is not None and receiver_room_id != "": res_data_message_token_user = send_message(url_server=matrix_server_url, @@ -250,18 +250,18 @@ def send_job_message( matrix_helper_logger.warning('a matrix message could not be sent to the token user as no personal room id was ' 'provided within the token') - for cc_receiver_room_id in cc_receivers_room_ids: - if cc_receiver_room_id is not None and cc_receiver_room_id != "": + for bcc_receiver_room_id in bcc_receivers_room_ids: + if bcc_receiver_room_id is not None and bcc_receiver_room_id != "": res_data_message_cc_user = send_message(url_server=matrix_server_url, sender_access_token=matrix_sender_access_token, - room_id=cc_receiver_room_id, + room_id=bcc_receiver_room_id, message_text=message_text, message_body_html=message_body_html ) message_data_cc_user = res_data_message_cc_user['message_data'] - message_data['message_data_cc_users'].append(message_data_cc_user) + message_data['message_data_bcc_users'].append(message_data_cc_user) res_content_cc_user = res_data_message_cc_user['res_content'] - res_content['res_content_cc_users'].append(res_content_cc_user) + res_content['res_content_bcc_users'].append(res_content_cc_user) store_status_matrix_message_info(message_data, status, scratch_dir, sending_time=sending_time, first_submitted_time=time_request) diff --git a/cdci_data_analysis/config_dir/conf_env.yml.example b/cdci_data_analysis/config_dir/conf_env.yml.example index f69fddd93..8664ab7ee 100644 --- a/cdci_data_analysis/config_dir/conf_env.yml.example +++ b/cdci_data_analysis/config_dir/conf_env.yml.example @@ -81,7 +81,7 @@ dispatcher: # access token of the sender account, from which the messages will be sent from matrix_sender_access_token: MATRIX_SENDER_ACCESS_TOKEN # list of additional room ids receivers towards which the message will be sent, besides the room id extracted from the token - matrix_cc_receivers_room_ids: ['room_id'] + matrix_bcc_receivers_room_ids: ['room_id'] # incident report related options, for the messages sent via matrix incident_report_matrix_options: # list of room ids receivers towards which the incident report message will be sent diff --git a/cdci_data_analysis/configurer.py b/cdci_data_analysis/configurer.py index f862b23a4..4ec6e6f1c 100644 --- a/cdci_data_analysis/configurer.py +++ b/cdci_data_analysis/configurer.py @@ -247,7 +247,7 @@ def __init__(self, cfg_dict, origin=None): disp_dict['email_options'].get('incident_report_email_options', {}).get('incident_report_receivers_email_addresses', None), disp_dict.get('matrix_options', {}).get('matrix_server_url', None), disp_dict.get('matrix_options', {}).get('matrix_sender_access_token', None), - disp_dict.get('matrix_options', {}).get('matrix_cc_receivers_room_ids', []), + disp_dict.get('matrix_options', {}).get('matrix_bcc_receivers_room_ids', []), disp_dict.get('matrix_options', {}).get('incident_report_matrix_options', {}).get('matrix_incident_report_receivers_room_ids', []), disp_dict.get('matrix_options', {}).get('incident_report_matrix_options', {}).get('matrix_incident_report_sender_personal_access_token', None), disp_dict.get('matrix_options', {}).get('matrix_message_sending_job_submitted', True), @@ -325,7 +325,7 @@ def set_conf_dispatcher(self, incident_report_receivers_email_addresses, matrix_server_url, matrix_sender_access_token, - matrix_cc_receivers_room_ids, + matrix_bcc_receivers_room_ids, matrix_incident_report_receivers_room_ids, matrix_incident_report_sender_personal_access_token, matrix_message_sending_job_submitted, @@ -375,7 +375,7 @@ def set_conf_dispatcher(self, self.incident_report_receivers_email_addresses = incident_report_receivers_email_addresses self.matrix_server_url = matrix_server_url self.matrix_sender_access_token = matrix_sender_access_token - self.matrix_cc_receivers_room_ids = matrix_cc_receivers_room_ids + self.matrix_bcc_receivers_room_ids = matrix_bcc_receivers_room_ids self.matrix_incident_report_receivers_room_ids = matrix_incident_report_receivers_room_ids self.matrix_incident_report_sender_personal_access_token = matrix_incident_report_sender_personal_access_token self.matrix_message_sending_job_submitted = matrix_message_sending_job_submitted diff --git a/cdci_data_analysis/pytest_fixtures.py b/cdci_data_analysis/pytest_fixtures.py index 9a5d26a90..3f8119714 100644 --- a/cdci_data_analysis/pytest_fixtures.py +++ b/cdci_data_analysis/pytest_fixtures.py @@ -564,7 +564,7 @@ def dispatcher_test_conf_with_matrix_options_fn(dispatcher_test_conf_fn): f.write('\n matrix_options:' '\n matrix_server_url: "https://matrix-client.matrix.org/"' f'\n matrix_sender_access_token: "{os.getenv("MATRIX_SENDER_ACCESS_TOKEN", "matrix_sender_access_token")}"' - f'\n matrix_cc_receivers_room_ids: ["{os.getenv("MATRIX_CC_RECEIVER_ROOM_ID", "")}"]' + f'\n matrix_bcc_receivers_room_ids: ["{os.getenv("MATRIX_CC_RECEIVER_ROOM_ID", "")}"]' '\n incident_report_matrix_options:' f'\n matrix_incident_report_receivers_room_ids: ["{os.getenv("MATRIX_INCIDENT_REPORT_RECEIVER_ROOM_ID", "matrix_incident_report_receivers_room_ids")}"]' f'\n matrix_incident_report_sender_personal_access_token: "{os.getenv("MATRIX_INCIDENT_REPORT_SENDER_PERSONAL_ACCESS_TOKEN", "matrix_incident_report_sender_personal_access_token")}"' @@ -578,7 +578,7 @@ def dispatcher_test_conf_with_matrix_options_fn(dispatcher_test_conf_fn): @pytest.fixture -def dispatcher_no_cc_matrix_room_ids(monkeypatch): +def dispatcher_no_bcc_matrix_room_ids(monkeypatch): monkeypatch.delenv('MATRIX_CC_RECEIVER_ROOM_ID', raising=False) diff --git a/tests/conftest.py b/tests/conftest.py index 846829e46..e8a35efa5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -14,7 +14,7 @@ dispatcher_long_living_fixture, gunicorn_dispatcher_long_living_fixture, dispatcher_long_living_fixture_with_matrix_options, - dispatcher_no_cc_matrix_room_ids, + dispatcher_no_bcc_matrix_room_ids, gunicorn_dispatcher_long_living_fixture_with_matrix_options, dispatcher_test_conf, dispatcher_test_conf_with_gallery, diff --git a/tests/test_matrix_messages.py b/tests/test_matrix_messages.py index 20d64fcdd..7fcf81b61 100644 --- a/tests/test_matrix_messages.py +++ b/tests/test_matrix_messages.py @@ -315,19 +315,19 @@ def test_matrix_message_run_analysis_callback(gunicorn_dispatcher_long_living_fi require_reference_matrix_message=True ) - assert 'res_content_cc_users' in matrix_message_status_details_obj['res_content'] - assert isinstance(matrix_message_status_details_obj['res_content']['res_content_cc_users'], list) - assert len(matrix_message_status_details_obj['res_content']['res_content_cc_users']) == 1 - assert 'event_id' in matrix_message_status_details_obj['res_content']['res_content_cc_users'][0] + assert 'res_content_bcc_users' in matrix_message_status_details_obj['res_content'] + assert isinstance(matrix_message_status_details_obj['res_content']['res_content_bcc_users'], list) + assert len(matrix_message_status_details_obj['res_content']['res_content_bcc_users']) == 1 + assert 'event_id' in matrix_message_status_details_obj['res_content']['res_content_bcc_users'][0] - matrix_cc_message_event_id_obj = matrix_message_status_details_obj['res_content']['res_content_cc_users'][0]['event_id'] + matrix_bcc_message_event_id_obj = matrix_message_status_details_obj['res_content']['res_content_bcc_users'][0]['event_id'] validate_matrix_message_content( - dispatcher_local_matrix_message_server.get_matrix_message_record(room_id=dispatcher_test_conf_with_matrix_options['matrix_options']['matrix_cc_receivers_room_ids'][0], - event_id=matrix_cc_message_event_id_obj), + dispatcher_local_matrix_message_server.get_matrix_message_record(room_id=dispatcher_test_conf_with_matrix_options['matrix_options']['matrix_bcc_receivers_room_ids'][0], + event_id=matrix_bcc_message_event_id_obj), 'submitted', - room_id=dispatcher_test_conf_with_matrix_options['matrix_options']['matrix_cc_receivers_room_ids'][0], - event_id=matrix_cc_message_event_id_obj, + room_id=dispatcher_test_conf_with_matrix_options['matrix_options']['matrix_bcc_receivers_room_ids'][0], + event_id=matrix_bcc_message_event_id_obj, user_id=token_payload['user_id'], dispatcher_job_state=dispatcher_job_state, variation_suffixes=["dummy"], @@ -460,9 +460,9 @@ def test_matrix_message_run_analysis_callback(gunicorn_dispatcher_long_living_fi assert 'matrix_message_status_details' in jdata matrix_message_status_details_obj = json.loads(jdata['matrix_message_status_details']) assert 'res_content' in matrix_message_status_details_obj - assert 'res_content_cc_users' in matrix_message_status_details_obj['res_content'] - assert isinstance(matrix_message_status_details_obj['res_content']['res_content_cc_users'], list) - assert len(matrix_message_status_details_obj['res_content']['res_content_cc_users']) == 1 + assert 'res_content_bcc_users' in matrix_message_status_details_obj['res_content'] + assert isinstance(matrix_message_status_details_obj['res_content']['res_content_bcc_users'], list) + assert len(matrix_message_status_details_obj['res_content']['res_content_bcc_users']) == 1 if request_cred == 'private-no-room-id': assert 'res_content_token_user' not in matrix_message_status_details_obj['res_content'] @@ -486,7 +486,7 @@ def test_matrix_message_run_analysis_callback(gunicorn_dispatcher_long_living_fi require_reference_matrix_message=True ) - matrix_cc_message_event_id_obj = matrix_message_status_details_obj['res_content']['res_content_cc_users'][0][ + matrix_bcc_message_event_id_obj = matrix_message_status_details_obj['res_content']['res_content_bcc_users'][0][ 'event_id'] # check the matrix message in the matrix message folders, and that the first one was produced @@ -494,11 +494,11 @@ def test_matrix_message_run_analysis_callback(gunicorn_dispatcher_long_living_fi validate_matrix_message_content( dispatcher_local_matrix_message_server.get_matrix_message_record( - room_id=dispatcher_test_conf_with_matrix_options['matrix_options']['matrix_cc_receivers_room_ids'][0], - event_id=matrix_cc_message_event_id_obj), + room_id=dispatcher_test_conf_with_matrix_options['matrix_options']['matrix_bcc_receivers_room_ids'][0], + event_id=matrix_bcc_message_event_id_obj), 'done', - room_id=dispatcher_test_conf_with_matrix_options['matrix_options']['matrix_cc_receivers_room_ids'][0], - event_id=matrix_cc_message_event_id_obj, + room_id=dispatcher_test_conf_with_matrix_options['matrix_options']['matrix_bcc_receivers_room_ids'][0], + event_id=matrix_bcc_message_event_id_obj, user_id=token_payload['user_id'], dispatcher_job_state=dispatcher_job_state, time_request_str=time_request_str, @@ -533,9 +533,9 @@ def test_matrix_message_run_analysis_callback(gunicorn_dispatcher_long_living_fi assert 'matrix_message_status_details' in jdata matrix_message_status_details_obj = json.loads(jdata['matrix_message_status_details']) assert 'res_content' in matrix_message_status_details_obj - assert 'res_content_cc_users' in matrix_message_status_details_obj['res_content'] - assert isinstance(matrix_message_status_details_obj['res_content']['res_content_cc_users'], list) - assert len(matrix_message_status_details_obj['res_content']['res_content_cc_users']) == 1 + assert 'res_content_bcc_users' in matrix_message_status_details_obj['res_content'] + assert isinstance(matrix_message_status_details_obj['res_content']['res_content_bcc_users'], list) + assert len(matrix_message_status_details_obj['res_content']['res_content_bcc_users']) == 1 if request_cred == 'private-no-room-id': assert 'res_content_token_user' not in matrix_message_status_details_obj['res_content'] else: @@ -612,10 +612,10 @@ def test_matrix_message_submitted_same_job(dispatcher_live_fixture_with_matrix_o assert 'res_content' in matrix_message_status_details_obj assert 'res_content_token_user' in matrix_message_status_details_obj['res_content'] assert 'event_id' in matrix_message_status_details_obj['res_content']['res_content_token_user'] - assert 'res_content_cc_users' in matrix_message_status_details_obj['res_content'] - assert isinstance(matrix_message_status_details_obj['res_content']['res_content_cc_users'], list) - assert len(matrix_message_status_details_obj['res_content']['res_content_cc_users']) == 1 - assert 'event_id' in matrix_message_status_details_obj['res_content']['res_content_cc_users'][0] + assert 'res_content_bcc_users' in matrix_message_status_details_obj['res_content'] + assert isinstance(matrix_message_status_details_obj['res_content']['res_content_bcc_users'], list) + assert len(matrix_message_status_details_obj['res_content']['res_content_bcc_users']) == 1 + assert 'event_id' in matrix_message_status_details_obj['res_content']['res_content_bcc_users'][0] time_request = jdata['time_request'] time_request_str = time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(float(time_request))) @@ -1206,7 +1206,7 @@ def test_incident_report(dispatcher_live_fixture_with_matrix_options, @pytest.mark.parametrize("default_values", [True, False]) @pytest.mark.parametrize("time_original_request_none", [False]) @pytest.mark.parametrize("request_cred", ['public', 'private', 'private-no-matrix-message', 'private-no-room-id']) -def test_matrix_message_run_analysis_callback_no_room_ids(dispatcher_no_cc_matrix_room_ids, +def test_matrix_message_run_analysis_callback_no_room_ids(dispatcher_no_bcc_matrix_room_ids, gunicorn_dispatcher_long_living_fixture_with_matrix_options, dispatcher_test_conf_with_matrix_options, dispatcher_local_matrix_message_server, @@ -1331,9 +1331,9 @@ def test_matrix_message_run_analysis_callback_no_room_ids(dispatcher_no_cc_matri require_reference_matrix_message=True ) - assert 'res_content_cc_users' in matrix_message_status_details_obj['res_content'] - assert isinstance(matrix_message_status_details_obj['res_content']['res_content_cc_users'], list) - assert len(matrix_message_status_details_obj['res_content']['res_content_cc_users']) == 0 + assert 'res_content_bcc_users' in matrix_message_status_details_obj['res_content'] + assert isinstance(matrix_message_status_details_obj['res_content']['res_content_bcc_users'], list) + assert len(matrix_message_status_details_obj['res_content']['res_content_bcc_users']) == 0 # for the call_back(s) in case the time of the original request is not provided if time_original_request_none: @@ -1457,9 +1457,9 @@ def test_matrix_message_run_analysis_callback_no_room_ids(dispatcher_no_cc_matri assert 'matrix_message_status_details' in jdata matrix_message_status_details_obj = json.loads(jdata['matrix_message_status_details']) assert 'res_content' in matrix_message_status_details_obj - assert 'res_content_cc_users' in matrix_message_status_details_obj['res_content'] - assert isinstance(matrix_message_status_details_obj['res_content']['res_content_cc_users'], list) - assert len(matrix_message_status_details_obj['res_content']['res_content_cc_users']) == 0 + assert 'res_content_bcc_users' in matrix_message_status_details_obj['res_content'] + assert isinstance(matrix_message_status_details_obj['res_content']['res_content_bcc_users'], list) + assert len(matrix_message_status_details_obj['res_content']['res_content_bcc_users']) == 0 if request_cred == 'private-no-room-id': assert 'res_content_token_user' not in matrix_message_status_details_obj['res_content'] @@ -1513,9 +1513,9 @@ def test_matrix_message_run_analysis_callback_no_room_ids(dispatcher_no_cc_matri assert 'matrix_message_status_details' in jdata matrix_message_status_details_obj = json.loads(jdata['matrix_message_status_details']) assert 'res_content' in matrix_message_status_details_obj - assert 'res_content_cc_users' in matrix_message_status_details_obj['res_content'] - assert isinstance(matrix_message_status_details_obj['res_content']['res_content_cc_users'], list) - assert len(matrix_message_status_details_obj['res_content']['res_content_cc_users']) == 0 + assert 'res_content_bcc_users' in matrix_message_status_details_obj['res_content'] + assert isinstance(matrix_message_status_details_obj['res_content']['res_content_bcc_users'], list) + assert len(matrix_message_status_details_obj['res_content']['res_content_bcc_users']) == 0 if request_cred == 'private-no-room-id': assert 'res_content_token_user' not in matrix_message_status_details_obj['res_content'] else: