From bbbae1a026609df5f582d70a228c9c6dca19aff1 Mon Sep 17 00:00:00 2001 From: Edvard Rejthar Date: Thu, 4 Apr 2024 15:25:21 +0200 Subject: [PATCH 1/2] ENH: smtp_batch invalid row hardening --- docs/user/bots.md | 4 ++-- intelmq/bots/outputs/smtp_batch/output.py | 12 +++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/docs/user/bots.md b/docs/user/bots.md index 791142dc2..6d44d7cf8 100644 --- a/docs/user/bots.md +++ b/docs/user/bots.md @@ -4969,7 +4969,7 @@ Note: The field "raw" gets base64 decoded if possible. Bytes `\n` and `\r` are r Launch it like this: ``` - cli [--tester tester's email] + --cli [--tester tester's email] ``` Example: ```bash @@ -4993,7 +4993,7 @@ You can schedule the batch sending easily with a cron script, I.E. put this into ``` # Send the e-mails every day at 6 AM -0 6 * * * /usr/local/bin/intelmq.bots.outputs.smtp_batch.output smtp-batch-output-cz cli --ignore-older-than-days 4 --send > /tmp/intelmq-send.log +0 6 * * * /usr/local/bin/intelmq.bots.outputs.smtp_batch.output smtp-batch-output-cz cli --ignore-older-than-days 4 --send &> /tmp/intelmq-send.log ``` **Module:** `intelmq.bots.outputs.smtp_batch.output` diff --git a/intelmq/bots/outputs/smtp_batch/output.py b/intelmq/bots/outputs/smtp_batch/output.py index 2a01b1ed2..66568f259 100644 --- a/intelmq/bots/outputs/smtp_batch/output.py +++ b/intelmq/bots/outputs/smtp_batch/output.py @@ -238,8 +238,7 @@ def prepare_mails(self): continue else: # visible both warning and print - print(f"Warning: {mail_record} timeout, too big to read from redis", flush=True) - self.logger.warning(f"Warning: {mail_record} timeout, too big to read from redis") + self.logger.warning(f"Warning: %s timeout, too big to read from redis", mail_record) self.timeout.append(mail_record) continue @@ -253,8 +252,11 @@ def prepare_mails(self): fieldnames = set() rows_output = [] for row in lines: - if threshold and row["time.observation"][:19] < threshold.isoformat()[:19]: - continue + try: + if threshold and row["time.observation"][:19] < threshold.isoformat()[:19]: + continue + except KeyError: + self.logger.warning(f"Warning: %s row skipped due to time.observation error", mail_record) fieldnames = fieldnames | set(row.keys()) keys = set(self.allowed_fieldnames).intersection(row) ordered_keys = [] @@ -290,7 +292,7 @@ def prepare_mails(self): try: zf.writestr(filename + ".csv", output.getvalue()) except Exception: - self.logger.error(f"Cannot zip mail {mail_record}") + self.logger.error(f"Cannot zip mail %s", mail_record) continue if email_to in self.alternative_mail: From 6e6c03b08f4f034428fb4d127c9b19d5dd97e909 Mon Sep 17 00:00:00 2001 From: Edvard Rejthar Date: Thu, 4 Apr 2024 16:23:11 +0200 Subject: [PATCH 2/2] [to-be-squashed] --- intelmq/bots/outputs/smtp_batch/output.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/intelmq/bots/outputs/smtp_batch/output.py b/intelmq/bots/outputs/smtp_batch/output.py index 66568f259..d24d11467 100644 --- a/intelmq/bots/outputs/smtp_batch/output.py +++ b/intelmq/bots/outputs/smtp_batch/output.py @@ -237,8 +237,7 @@ def prepare_mails(self): print("... failed ...", flush=True) continue else: - # visible both warning and print - self.logger.warning(f"Warning: %s timeout, too big to read from redis", mail_record) + self.logger.warning("Warning: timeout, too big to read from redis: %r", mail_record) self.timeout.append(mail_record) continue @@ -256,7 +255,7 @@ def prepare_mails(self): if threshold and row["time.observation"][:19] < threshold.isoformat()[:19]: continue except KeyError: - self.logger.warning(f"Warning: %s row skipped due to time.observation error", mail_record) + self.logger.warning("Warning: row skipped due to time.observation error: %r", mail_record) fieldnames = fieldnames | set(row.keys()) keys = set(self.allowed_fieldnames).intersection(row) ordered_keys = [] @@ -292,7 +291,7 @@ def prepare_mails(self): try: zf.writestr(filename + ".csv", output.getvalue()) except Exception: - self.logger.error(f"Cannot zip mail %s", mail_record) + self.logger.error("Error: Cannot zip mail: %r", mail_record) continue if email_to in self.alternative_mail: