Add Header and footer text

dag_confs/examples_and_tests/header_and_footer_example.yaml

@@ -0,0 +1,21 @@
+dag:
+  id: header_and_footer_example
+  description: DAG de teste
+  tags:
+    - dou
+  schedule: 0 8 * * MON-FRI
+  search:
+    sources:
+    - DOU
+    terms:
+    - tecnologia
+    - informação
+    department:
+      - Ministério da Gestão e da Inovação em Serviços Públicos
+      - Ministério da Defesa
+  report:
+    emails:
+      - [email protected]
+    subject: "Teste do Ro-dou"
+    header_text: <p><strong>Greetings<strong></p>
+    footer_text: <p>Best Regards</p>

schemas/ro-dou.json

@@ -224,6 +224,14 @@
             "hide_filters": {
               "type": "boolean",
               "description": "description"
+            },
+            "header_text": {
+              "type": "string",
+              "description": "description"
+            },
+            "footer_text": {
+              "type": "string",
+              "description": "description"
             }
           },
           "additionalProperties": false

src/notification/discord_sender.py

@@ -1,5 +1,5 @@
 import requests
-
+import re
 from notification.isender import ISender
 
 
@@ -9,9 +9,15 @@ class DiscordSender(ISender):
     def __init__(self, specs) -> None:
         self.webhook_url = specs.discord_webhook
         self.hide_filters = specs.hide_filters
+        self.header_text = specs.header_text
+        self.footer_text = specs.footer_text
 
     def send(self, search_report: list, report_date: str = None):
         """Parse the content, and send message to Discord"""
+        if self.header_text:
+            header_text = self._remove_html_tags(self.header_text)
+            self.send_text(header_text)
+
         for search in search_report:
             if search["header"]:
                 self.send_text(f'**{search["header"]}**')
@@ -31,6 +37,10 @@ def send(self, search_report: list, report_date: str = None):
                         "**Nenhum dos termos pesquisados foi encontrado nesta consulta**"
                     )
 
+        if self.footer_text:
+            footer_text = self._remove_html_tags(self.footer_text)
+            self.send_text(footer_text)
+
     def send_text(self, content):
         self.send_data({"content": content})
 
@@ -52,3 +62,9 @@ def send_data(self, data):
         data["username"] = "Querido Prisma (rodou)"
         result = requests.post(self.webhook_url, json=data)
         result.raise_for_status()
+
+    def _remove_html_tags(self, text):
+        # Define a regular expression pattern to match HTML tags
+        clean = re.compile('<.*?>')
+        # Substitute HTML tags with an empty string
+        return re.sub(clean, '', text)

src/notification/email_sender.py

@@ -70,6 +70,9 @@ def generate_email_content(self) -> str:
         with open(file_path, "r") as f:
             blocks = [f"<style>\n{f.read()}</style>"]
 
+        if self.specs.header_text:
+            blocks.append(self.specs.header_text)
+
         for search in self.search_report:
 
             if search["header"]:
@@ -121,6 +124,8 @@ def generate_email_content(self) -> str:
                                 blocks.append(textwrap.dedent(item_html))
 
             blocks.append("---")
+            if self.specs.footer_text:
+                blocks.append(self.specs.footer_text)
 
         return markdown.markdown("\n".join(blocks))
 

src/notification/slack_sender.py

@@ -1,7 +1,7 @@
 from datetime import datetime
 
 import requests
-
+import re
 from notification.isender import ISender
 
 
@@ -12,9 +12,15 @@ def __init__(self, specs) -> None:
         self.webhook_url = specs.slack_webhook
         self.blocks = []
         self.hide_filters = specs.hide_filters
+        self.header_text = specs.header_text
+        self.footer_text = specs.footer_text
 
     def send(self, search_report: list, report_date: str = None):
         """Parse the content, and send message to Slack"""
+        if self.header_text:
+            header_text = _remove_html_tags(self.header_text)
+            self._add_header(header_text)
+
         for search in search_report:
             if search["header"]:
                 self._add_header(search["header"])
@@ -33,6 +39,10 @@ def send(self, search_report: list, report_date: str = None):
                     self._add_text(
                         "Nenhum dos termos pesquisados foi encontrado nesta consulta."
                     )
+
+        if self.footer_text:
+            footer_text = _remove_html_tags(self.footer_text)
+            self._add_header(footer_text)
         self._flush()
 
     def _add_header(self, text):
@@ -99,3 +109,9 @@ def _format_date(date_str: str) -> str:
     date = datetime.strptime(date_str, "%d/%m/%Y")
     _from, _to = WEEKDAYS_EN_TO_PT[date.weekday()]
     return date.strftime("%a %d/%m").replace(_from, _to)
+
+def _remove_html_tags(text):
+    # Define a regular expression pattern to match HTML tags
+    clean = re.compile('<.*?>')
+    # Substitute HTML tags with an empty string
+    return re.sub(clean, '', text)

src/parsers.py

@@ -45,6 +45,8 @@ class DAGConfig:
     dag_tags: Set[str]
     owner: str
     hide_filters: bool
+    header_text: str
+    footer_text: str
 
 
 class FileParser(ABC):
@@ -154,6 +156,8 @@ def _parse_yaml(self) -> DAGConfig:
         subject = report.get("subject", "Extraçao do DOU")
         attach_csv = report.get("attach_csv", False)
         hide_filters = report.get("hide_filters", False)
+        header_text = report.get("header_text", None)
+        footer_text = report.get("footer_text", None)
 
         return DAGConfig(
             dag_id=dag_id,
@@ -169,7 +173,9 @@ def _parse_yaml(self) -> DAGConfig:
             doc_md=doc_md,
             dag_tags=set(dag_tags),
             owner=owner,
-            hide_filters=hide_filters
+            hide_filters=hide_filters,
+            header_text=header_text,
+            footer_text=footer_text
         )
 
     def _get_terms_params(self, search) -> Tuple[List[str], str, str]:

tests/discord_sender_test.py

@@ -8,8 +8,12 @@
 
 @pytest.fixture
 def mocked_specs():
-    Specs = namedtuple('Specs', ['discord_webhook', 'hide_filters'])
-    return Specs(WEBHOOK, False)
+    Specs = namedtuple('Specs',
+                       ['discord_webhook',
+                        'hide_filters',
+                        'header_text',
+                        'footer_text'])
+    return Specs(WEBHOOK, False, None, None)
 
 def test_send_discord_data(session_mocker: MockerFixture, mocked_specs):
     session_mocker.patch(

tests/parsers_test.py

@@ -69,6 +69,8 @@ def test_hash_dag_id(yaml_parser, dag_id, size, hashed):
                 "dag_tags": {"dou", "generated_dag"},
                 "owner": "",
                 "hide_filters": False,
+                "header_text": None,
+                "footer_text": None,
             },
         ),
         (
@@ -109,6 +111,8 @@ def test_hash_dag_id(yaml_parser, dag_id, size, hashed):
                 "dag_tags": {"dou", "generated_dag", "projeto_a", "departamento_x"},
                 "owner": "pessoa 1, pessoa 2",
                 "hide_filters": False,
+                "header_text": None,
+                "footer_text": None,
             },
         ),
         (
@@ -152,6 +156,8 @@ def test_hash_dag_id(yaml_parser, dag_id, size, hashed):
                 "dag_tags": {"dou", "generated_dag"},
                 "owner": "",
                 "hide_filters": False,
+                "header_text": None,
+                "footer_text": None,
             },
         ),
         (
@@ -188,6 +194,8 @@ def test_hash_dag_id(yaml_parser, dag_id, size, hashed):
                 "dag_tags": {"dou", "generated_dag"},
                 "owner": "",
                 "hide_filters": False,
+                "header_text": None,
+                "footer_text": None,
             },
         ),
         (
@@ -236,6 +244,8 @@ def test_hash_dag_id(yaml_parser, dag_id, size, hashed):
                 "dag_tags": {"dou", "generated_dag"},
                 "owner": "",
                 "hide_filters": False,
+                "header_text": None,
+                "footer_text": None,
             },
         ),
         (
@@ -275,6 +285,8 @@ def test_hash_dag_id(yaml_parser, dag_id, size, hashed):
                 "dag_tags": {"dou", "generated_dag"},
                 "owner": "",
                 "hide_filters": False,
+                "header_text": None,
+                "footer_text": None,
             },
         ),
         (
@@ -311,6 +323,8 @@ def test_hash_dag_id(yaml_parser, dag_id, size, hashed):
                 "dag_tags": {"dou", "generated_dag", "inlabs"},
                 "owner": "cdata",
                 "hide_filters": False,
+                "header_text": None,
+                "footer_text": None,
             },
         ),
         (
@@ -350,6 +364,8 @@ def test_hash_dag_id(yaml_parser, dag_id, size, hashed):
                 "dag_tags": {"dou", "generated_dag", "inlabs"},
                 "owner": "cdata",
                 "hide_filters": False,
+                "header_text": None,
+                "footer_text": None,
             },
         ),
         (
@@ -410,6 +426,8 @@ def test_hash_dag_id(yaml_parser, dag_id, size, hashed):
                 "dag_tags": {"dou", "generated_dag", "inlabs"},
                 "owner": "",
                 "hide_filters": False,
+                "header_text": None,
+                "footer_text": None,
             },
         ),
         (
@@ -449,10 +467,54 @@ def test_hash_dag_id(yaml_parser, dag_id, size, hashed):
                 "dag_tags": {"dou", "inlabs", "generated_dag"},
                 "owner": "",
                 "hide_filters": True,
+                "header_text": None,
+                "footer_text": None,
+            },
+        ),
+        (
+            "header_and_footer_example.yaml",
+            {
+                "dag_id": "header_and_footer_example",
+                "search": [
+                    {
+                        "terms": ["tecnologia", "informação"],
+                        "header": None,
+                        "sources": ["DOU"],
+                        "sql": None,
+                        "conn_id": None,
+                        "territory_id": None,
+                        "dou_sections": ["TODOS"],
+                        "search_date": "DIA",
+                        "field": "TUDO",
+                        "is_exact_search": True,
+                        "ignore_signature_match": False,
+                        "force_rematch": None,
+                        "full_text": None,
+                        "department": [
+                            "Ministério da Gestão e da Inovação em Serviços Públicos",
+                            "Ministério da Defesa",
+                        ],
+                    }
+                ],
+                "emails": ["[email protected]"],
+                "subject": "Teste do Ro-dou",
+                "attach_csv": False,
+                "discord_webhook": None,
+                "slack_webhook": None,
+                "schedule": "0 8 * * MON-FRI",
+                "description": "DAG de teste",
+                "skip_null": True,
+                "doc_md": None,
+                "dag_tags": {"dou", "generated_dag"},
+                "owner": "",
+                "hide_filters": False,
+                "header_text": "<p><strong>Greetings<strong></p>",
+                "footer_text": "<p>Best Regards</p>",
             },
         ),
     ],
 )
+
 def test_parse(filepath, result_tuple):
     filepath = os.path.join(
         DouDigestDagGenerator().YAMLS_DIR, "examples_and_tests", filepath