Skip to content

Commit

Permalink
Merge remote-tracking branch 'refs/remotes/origin/main'
Browse files Browse the repository at this point in the history
  • Loading branch information
marcelosinnerworkings committed Jun 21, 2024
2 parents 08301cd + dee5dd5 commit 4e96a94
Show file tree
Hide file tree
Showing 9 changed files with 145 additions and 6 deletions.
21 changes: 21 additions & 0 deletions dag_confs/examples_and_tests/header_and_footer_example.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
dag:
id: header_and_footer_example
description: DAG de teste
tags:
- dou
schedule: 0 8 * * MON-FRI
search:
sources:
- DOU
terms:
- tecnologia
- informação
department:
- Ministério da Gestão e da Inovação em Serviços Públicos
- Ministério da Defesa
report:
emails:
- [email protected]
subject: "Teste do Ro-dou"
header_text: <p><strong>Greetings<strong></p>
footer_text: <p>Best Regards</p>
8 changes: 8 additions & 0 deletions schemas/ro-dou.json
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,14 @@
"hide_filters": {
"type": "boolean",
"description": "description"
},
"header_text": {
"type": "string",
"description": "description"
},
"footer_text": {
"type": "string",
"description": "description"
}
},
"additionalProperties": false
Expand Down
18 changes: 17 additions & 1 deletion src/notification/discord_sender.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import requests

import re
from notification.isender import ISender


Expand All @@ -9,9 +9,15 @@ class DiscordSender(ISender):
def __init__(self, specs) -> None:
self.webhook_url = specs.discord_webhook
self.hide_filters = specs.hide_filters
self.header_text = specs.header_text
self.footer_text = specs.footer_text

def send(self, search_report: list, report_date: str = None):
"""Parse the content, and send message to Discord"""
if self.header_text:
header_text = self._remove_html_tags(self.header_text)
self.send_text(header_text)

for search in search_report:
if search["header"]:
self.send_text(f'**{search["header"]}**')
Expand All @@ -31,6 +37,10 @@ def send(self, search_report: list, report_date: str = None):
"**Nenhum dos termos pesquisados foi encontrado nesta consulta**"
)

if self.footer_text:
footer_text = self._remove_html_tags(self.footer_text)
self.send_text(footer_text)

def send_text(self, content):
self.send_data({"content": content})

Expand All @@ -52,3 +62,9 @@ def send_data(self, data):
data["username"] = "Querido Prisma (rodou)"
result = requests.post(self.webhook_url, json=data)
result.raise_for_status()

def _remove_html_tags(self, text):
# Define a regular expression pattern to match HTML tags
clean = re.compile('<.*?>')
# Substitute HTML tags with an empty string
return re.sub(clean, '', text)
7 changes: 6 additions & 1 deletion src/notification/email_sender.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,9 @@ def generate_email_content(self) -> str:
with open(file_path, "r") as f:
blocks = [f"<style>\n{f.read()}</style>"]

if self.specs.header_text:
blocks.append(self.specs.header_text)

for search in self.search_report:

if search["header"]:
Expand Down Expand Up @@ -120,7 +123,9 @@ def generate_email_content(self) -> str:
item_html += f"<p class='abstract-marker'>{item['abstract']}</p><br><br>"
blocks.append(textwrap.dedent(item_html))

blocks.append("---")
blocks.append("---")
if self.specs.footer_text:
blocks.append(self.specs.footer_text)

return markdown.markdown("\n".join(blocks))

Expand Down
18 changes: 17 additions & 1 deletion src/notification/slack_sender.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from datetime import datetime

import requests

import re
from notification.isender import ISender


Expand All @@ -12,9 +12,15 @@ def __init__(self, specs) -> None:
self.webhook_url = specs.slack_webhook
self.blocks = []
self.hide_filters = specs.hide_filters
self.header_text = specs.header_text
self.footer_text = specs.footer_text

def send(self, search_report: list, report_date: str = None):
"""Parse the content, and send message to Slack"""
if self.header_text:
header_text = _remove_html_tags(self.header_text)
self._add_header(header_text)

for search in search_report:
if search["header"]:
self._add_header(search["header"])
Expand All @@ -33,6 +39,10 @@ def send(self, search_report: list, report_date: str = None):
self._add_text(
"Nenhum dos termos pesquisados foi encontrado nesta consulta."
)

if self.footer_text:
footer_text = _remove_html_tags(self.footer_text)
self._add_header(footer_text)
self._flush()

def _add_header(self, text):
Expand Down Expand Up @@ -99,3 +109,9 @@ def _format_date(date_str: str) -> str:
date = datetime.strptime(date_str, "%d/%m/%Y")
_from, _to = WEEKDAYS_EN_TO_PT[date.weekday()]
return date.strftime("%a %d/%m").replace(_from, _to)

def _remove_html_tags(text):
# Define a regular expression pattern to match HTML tags
clean = re.compile('<.*?>')
# Substitute HTML tags with an empty string
return re.sub(clean, '', text)
8 changes: 7 additions & 1 deletion src/parsers.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ class DAGConfig:
dag_tags: Set[str]
owner: str
hide_filters: bool
header_text: str
footer_text: str


class FileParser(ABC):
Expand Down Expand Up @@ -154,6 +156,8 @@ def _parse_yaml(self) -> DAGConfig:
subject = report.get("subject", "Extraçao do DOU")
attach_csv = report.get("attach_csv", False)
hide_filters = report.get("hide_filters", False)
header_text = report.get("header_text", None)
footer_text = report.get("footer_text", None)

return DAGConfig(
dag_id=dag_id,
Expand All @@ -169,7 +173,9 @@ def _parse_yaml(self) -> DAGConfig:
doc_md=doc_md,
dag_tags=set(dag_tags),
owner=owner,
hide_filters=hide_filters
hide_filters=hide_filters,
header_text=header_text,
footer_text=footer_text
)

def _get_terms_params(self, search) -> Tuple[List[str], str, str]:
Expand Down
1 change: 1 addition & 0 deletions src/report_style.css
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
* {
font-family: 'rawline',sans-serif;
text-align: justify;
}
a:link {
color: black;
Expand Down
8 changes: 6 additions & 2 deletions tests/discord_sender_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,12 @@

@pytest.fixture
def mocked_specs():
Specs = namedtuple('Specs', ['discord_webhook', 'hide_filters'])
return Specs(WEBHOOK, False)
Specs = namedtuple('Specs',
['discord_webhook',
'hide_filters',
'header_text',
'footer_text'])
return Specs(WEBHOOK, False, None, None)

def test_send_discord_data(session_mocker: MockerFixture, mocked_specs):
session_mocker.patch(
Expand Down
62 changes: 62 additions & 0 deletions tests/parsers_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,8 @@ def test_hash_dag_id(yaml_parser, dag_id, size, hashed):
"dag_tags": {"dou", "generated_dag"},
"owner": "",
"hide_filters": False,
"header_text": None,
"footer_text": None,
},
),
(
Expand Down Expand Up @@ -109,6 +111,8 @@ def test_hash_dag_id(yaml_parser, dag_id, size, hashed):
"dag_tags": {"dou", "generated_dag", "projeto_a", "departamento_x"},
"owner": "pessoa 1, pessoa 2",
"hide_filters": False,
"header_text": None,
"footer_text": None,
},
),
(
Expand Down Expand Up @@ -152,6 +156,8 @@ def test_hash_dag_id(yaml_parser, dag_id, size, hashed):
"dag_tags": {"dou", "generated_dag"},
"owner": "",
"hide_filters": False,
"header_text": None,
"footer_text": None,
},
),
(
Expand Down Expand Up @@ -188,6 +194,8 @@ def test_hash_dag_id(yaml_parser, dag_id, size, hashed):
"dag_tags": {"dou", "generated_dag"},
"owner": "",
"hide_filters": False,
"header_text": None,
"footer_text": None,
},
),
(
Expand Down Expand Up @@ -236,6 +244,8 @@ def test_hash_dag_id(yaml_parser, dag_id, size, hashed):
"dag_tags": {"dou", "generated_dag"},
"owner": "",
"hide_filters": False,
"header_text": None,
"footer_text": None,
},
),
(
Expand Down Expand Up @@ -275,6 +285,8 @@ def test_hash_dag_id(yaml_parser, dag_id, size, hashed):
"dag_tags": {"dou", "generated_dag"},
"owner": "",
"hide_filters": False,
"header_text": None,
"footer_text": None,
},
),
(
Expand Down Expand Up @@ -311,6 +323,8 @@ def test_hash_dag_id(yaml_parser, dag_id, size, hashed):
"dag_tags": {"dou", "generated_dag", "inlabs"},
"owner": "cdata",
"hide_filters": False,
"header_text": None,
"footer_text": None,
},
),
(
Expand Down Expand Up @@ -350,6 +364,8 @@ def test_hash_dag_id(yaml_parser, dag_id, size, hashed):
"dag_tags": {"dou", "generated_dag", "inlabs"},
"owner": "cdata",
"hide_filters": False,
"header_text": None,
"footer_text": None,
},
),
(
Expand Down Expand Up @@ -410,6 +426,8 @@ def test_hash_dag_id(yaml_parser, dag_id, size, hashed):
"dag_tags": {"dou", "generated_dag", "inlabs"},
"owner": "",
"hide_filters": False,
"header_text": None,
"footer_text": None,
},
),
(
Expand Down Expand Up @@ -449,10 +467,54 @@ def test_hash_dag_id(yaml_parser, dag_id, size, hashed):
"dag_tags": {"dou", "inlabs", "generated_dag"},
"owner": "",
"hide_filters": True,
"header_text": None,
"footer_text": None,
},
),
(
"header_and_footer_example.yaml",
{
"dag_id": "header_and_footer_example",
"search": [
{
"terms": ["tecnologia", "informação"],
"header": None,
"sources": ["DOU"],
"sql": None,
"conn_id": None,
"territory_id": None,
"dou_sections": ["TODOS"],
"search_date": "DIA",
"field": "TUDO",
"is_exact_search": True,
"ignore_signature_match": False,
"force_rematch": None,
"full_text": None,
"department": [
"Ministério da Gestão e da Inovação em Serviços Públicos",
"Ministério da Defesa",
],
}
],
"emails": ["[email protected]"],
"subject": "Teste do Ro-dou",
"attach_csv": False,
"discord_webhook": None,
"slack_webhook": None,
"schedule": "0 8 * * MON-FRI",
"description": "DAG de teste",
"skip_null": True,
"doc_md": None,
"dag_tags": {"dou", "generated_dag"},
"owner": "",
"hide_filters": False,
"header_text": "<p><strong>Greetings<strong></p>",
"footer_text": "<p>Best Regards</p>",
},
),
],
)

def test_parse(filepath, result_tuple):
filepath = os.path.join(
DouDigestDagGenerator().YAMLS_DIR, "examples_and_tests", filepath
Expand Down

0 comments on commit 4e96a94

Please sign in to comment.