From 5195d796d414d8f503cc0ef3393bcee474c868dc Mon Sep 17 00:00:00 2001 From: Jozsef K <43893201+nimshi89@users.noreply.github.com> Date: Thu, 27 Jun 2024 08:50:44 +0100 Subject: [PATCH 01/11] added logging stuff --- dpytools/logging/utility.py | 73 +++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/dpytools/logging/utility.py b/dpytools/logging/utility.py index cc24dec..fedb049 100644 --- a/dpytools/logging/utility.py +++ b/dpytools/logging/utility.py @@ -1,7 +1,11 @@ import sys import traceback +from requests import Response +from datetime import datetime, timedelta from typing import Dict, List +from urllib.parse import urlparse + def level_to_severity(level: int) -> int: """ @@ -47,3 +51,72 @@ def create_error_dict(error: Exception) -> List[Dict]: # Listify in keeping with expected DP logging structures return [error_dict] + +def get_scheme(url: str) -> str: + """This function will return the scheme from the provided url.""" + + index = url.find('/') + scheme = url[0:index-1] + return scheme + +def get_domain(url: str) -> str: + """This funciton will return the domain name from the provided url.""" + #Parsing url to extract the domain name + parsed_url = urlparse(url) + + #Getting the domain name + domain = parsed_url.netloc.split(':')[0] + return domain + +def get_port(url: str) ->int: + """This funciton will return the port number form the provided url.""" + #Parsing url + parsed_url = urlparse(url) + + #checking if the port was give if not scheking scheme for port number + if parsed_url.port is None: + if parsed_url.scheme == "http": + return 80 + else: + return 443 + else: + return parsed_url.port + +def get_start_date(date: str) ->str: + + strp_time = datetime.strptime(date, '%a, %d %b %Y %H:%M:%S GMT') + + return strp_time.isoformat() + "Z" + +def get_end_date(time_delta: timedelta, date: str) ->str: + """This function will calculate the end_date by addint the duration to the start date.""" + + strp_time = datetime.strptime(date, '%a, %d %b %Y %H:%M:%S GMT') + td = timedelta(microseconds=time_delta.microseconds) + + end_date = strp_time + td + + return end_date.isoformat() + "Z" + +def calculate_duration_in_nanoseconds(time_delta: timedelta, date: str)->int: + """This funciton will convert the duration from Miliseconds to Nanoseconds.""" + + strp_time = datetime.strptime(date, '%a, %d %b %Y %H:%M:%S GMT') + td = timedelta(microseconds=time_delta.microseconds) + + end_date = strp_time + td + + duration = end_date - strp_time + + return duration.microseconds * 1000 + +def get_content_length(res: Response)->int: + """ + This funciton will try to get the 'Content-Lenght' + if there is noone it will return a default 0. + """ + try: + content_length = res.headers["Content-Length"] + return int(content_length) + except KeyError: + return 0 From 53126482e23b1eaeb89b8f1fbc226e1e6cfe0264 Mon Sep 17 00:00:00 2001 From: Jozsef K <43893201+nimshi89@users.noreply.github.com> Date: Thu, 27 Jun 2024 08:54:42 +0100 Subject: [PATCH 02/11] f --- dpytools/logging/logger.py | 47 ++++++++++++++++++++++++++++--------- dpytools/logging/utility.py | 2 +- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/dpytools/logging/logger.py b/dpytools/logging/logger.py index 7fd8d2d..717134b 100644 --- a/dpytools/logging/logger.py +++ b/dpytools/logging/logger.py @@ -1,13 +1,22 @@ import logging import os import sys +import requests from datetime import datetime, timezone from typing import Dict, List, Optional import structlog -from dpytools.logging.utility import create_error_dict, level_to_severity - +from dpytools.logging.utility import ( + create_error_dict, + level_to_severity, + get_domain, + get_scheme, + get_domain, + get_port, + get_start_date, + get_end_date, + calculate_duration_in_nanoseconds) class DpLogger: def __init__(self, namespace: str): @@ -56,6 +65,7 @@ def _log( error: Optional[List] = None, data: Optional[Dict] = None, raw: str = None, + response = Optional[requests.Response], ): data_dict = data if data is not None else {} data_dict["level"] = logging.getLevelName(level) @@ -74,12 +84,27 @@ def _log( "errors": create_error_dict(error) if error is not None else None, } + r_dict = { + "method": response.request.method, + "scheme": get_scheme(response.url), + "host": get_domain(response.url), + "port": get_port(response.url), + "path": response.request.path_url, + "status_code" : response.status_code, + "started_at":get_start_date(response.headers["Date"]), + "ended_at":get_end_date(response.elapsed, response.headers["Date"]), + "duration": calculate_duration_in_nanoseconds(response.elapsed, response.headers["Date"]), + "response_content_length":len(response.content) + } + + reponse_dict = r_dict if response is not None else {} + self._logger.log(**log_event) if self.flush_stdout_after_log_entry is True: sys.stdout.flush() - def debug(self, event: str, raw: str = None, data: Dict = None): + def debug(self, event: str, raw: str = None, data: Dict = None, response: requests.Response = None): """ Log at the debug level. @@ -87,9 +112,9 @@ def debug(self, event: str, raw: str = None, data: Dict = None): raw : a raw string of any log messages captured for a third party library data : arbitrary key-value pairs that may be of use in providing context """ - self._log(event, 10, raw=raw, data=data) + self._log(event, 10, raw=raw, data=data, response=response) - def info(self, event: str, raw: str = None, data: Dict = None): + def info(self, event: str, raw: str = None, data: Dict = None, response: requests.Response = None): """ Log at the info level. @@ -99,7 +124,7 @@ def info(self, event: str, raw: str = None, data: Dict = None): """ self._log(event, 20, raw=raw, data=data) - def warning(self, event: str, raw: str = None, data: Dict = None): + def warning(self, event: str, raw: str = None, data: Dict = None, response: requests.Response = None): """ Log at the warning level. @@ -107,9 +132,9 @@ def warning(self, event: str, raw: str = None, data: Dict = None): raw : a raw string of any log messages captured for a third party library data : arbitrary key-value pairs that may be of use in providing context """ - self._log(event, 30, raw=raw, data=data) + self._log(event, 30, raw=raw, data=data, response=response) - def error(self, event: str, error: Exception, raw: str = None, data: Dict = None): + def error(self, event: str, error: Exception, raw: str = None, data: Dict = None, response: requests.Response = None): """ Log at the error level. @@ -118,10 +143,10 @@ def error(self, event: str, error: Exception, raw: str = None, data: Dict = None raw : a raw string of any log messages captured for a third party library data : arbitrary key-value pairs that may be of use in providing context """ - self._log(event, 40, error=error, raw=raw, data=data) + self._log(event, 40, error=error, raw=raw, data=data, response=response) def critical( - self, event: str, error: Exception, raw: str = None, data: Dict = None + self, event: str, error: Exception, raw: str = None, data: Dict = None, response: requests.Response = None ): """ IMPORTANT: You should only be logging at the critical level during @@ -135,4 +160,4 @@ def critical( raw : a raw string of any log messages captured for a third party library data : arbitrary key-value pairs that may be of use in providing context """ - self._log(event, 50, error=error, raw=raw, data=data) + self._log(event, 50, error=error, raw=raw, data=data, response=response) diff --git a/dpytools/logging/utility.py b/dpytools/logging/utility.py index fedb049..48428a2 100644 --- a/dpytools/logging/utility.py +++ b/dpytools/logging/utility.py @@ -1,10 +1,10 @@ import sys import traceback -from requests import Response from datetime import datetime, timedelta from typing import Dict, List from urllib.parse import urlparse +from requests import Response def level_to_severity(level: int) -> int: From 48759ed0e0047d49722ae0f1395210f830a44809 Mon Sep 17 00:00:00 2001 From: Jozsef K <43893201+nimshi89@users.noreply.github.com> Date: Thu, 27 Jun 2024 08:58:21 +0100 Subject: [PATCH 03/11] f --- dpytools/logging/logger.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/dpytools/logging/logger.py b/dpytools/logging/logger.py index 717134b..f9c3191 100644 --- a/dpytools/logging/logger.py +++ b/dpytools/logging/logger.py @@ -1,22 +1,22 @@ import logging import os import sys -import requests from datetime import datetime, timezone from typing import Dict, List, Optional import structlog +import requests + from dpytools.logging.utility import ( - create_error_dict, - level_to_severity, - get_domain, - get_scheme, + calculate_duration_in_nanoseconds, + create_error_dict, get_domain, + get_end_date, get_port, + get_scheme, get_start_date, - get_end_date, - calculate_duration_in_nanoseconds) + level_to_severity) class DpLogger: def __init__(self, namespace: str): From 79b7155aed2dde7f9414cf739ff13a7e50a20c1c Mon Sep 17 00:00:00 2001 From: Jozsef K <43893201+nimshi89@users.noreply.github.com> Date: Thu, 27 Jun 2024 09:00:16 +0100 Subject: [PATCH 04/11] f --- dpytools/logging/logger.py | 2 +- dpytools/logging/utility.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dpytools/logging/logger.py b/dpytools/logging/logger.py index f9c3191..ed7ee3d 100644 --- a/dpytools/logging/logger.py +++ b/dpytools/logging/logger.py @@ -4,8 +4,8 @@ from datetime import datetime, timezone from typing import Dict, List, Optional -import structlog import requests +import structlog from dpytools.logging.utility import ( diff --git a/dpytools/logging/utility.py b/dpytools/logging/utility.py index 48428a2..0a5dae3 100644 --- a/dpytools/logging/utility.py +++ b/dpytools/logging/utility.py @@ -2,8 +2,8 @@ import traceback from datetime import datetime, timedelta from typing import Dict, List - from urllib.parse import urlparse + from requests import Response From 2c3013bbd53a840329ad7d4660b01bedf6f6ad6b Mon Sep 17 00:00:00 2001 From: Jozsef K <43893201+nimshi89@users.noreply.github.com> Date: Thu, 27 Jun 2024 09:01:57 +0100 Subject: [PATCH 05/11] f --- dpytools/logging/logger.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dpytools/logging/logger.py b/dpytools/logging/logger.py index ed7ee3d..86c1277 100644 --- a/dpytools/logging/logger.py +++ b/dpytools/logging/logger.py @@ -10,13 +10,14 @@ from dpytools.logging.utility import ( calculate_duration_in_nanoseconds, - create_error_dict, + create_error_dict, get_domain, get_end_date, get_port, get_scheme, get_start_date, - level_to_severity) + level_to_severity +) class DpLogger: def __init__(self, namespace: str): From eda06cfb8932650353b9003dfcded6297984570b Mon Sep 17 00:00:00 2001 From: Jozsef K <43893201+nimshi89@users.noreply.github.com> Date: Thu, 27 Jun 2024 09:05:24 +0100 Subject: [PATCH 06/11] f --- dpytools/logging/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpytools/logging/logger.py b/dpytools/logging/logger.py index 86c1277..a3992ec 100644 --- a/dpytools/logging/logger.py +++ b/dpytools/logging/logger.py @@ -16,7 +16,7 @@ get_port, get_scheme, get_start_date, - level_to_severity + level_to_severity, ) class DpLogger: From 900fb0636755b255a19081b8581092e5ba6c8e37 Mon Sep 17 00:00:00 2001 From: Jozsef K <43893201+nimshi89@users.noreply.github.com> Date: Thu, 27 Jun 2024 09:13:30 +0100 Subject: [PATCH 07/11] f --- dpytools/logging/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dpytools/logging/logger.py b/dpytools/logging/logger.py index a3992ec..5c41bf0 100644 --- a/dpytools/logging/logger.py +++ b/dpytools/logging/logger.py @@ -7,7 +7,6 @@ import requests import structlog - from dpytools.logging.utility import ( calculate_duration_in_nanoseconds, create_error_dict, @@ -19,6 +18,7 @@ level_to_severity, ) + class DpLogger: def __init__(self, namespace: str): """ From 9ecf8b7ae2e9ba61ca007e75e0e1b17e8e45e24b Mon Sep 17 00:00:00 2001 From: Jozsef K Date: Thu, 12 Sep 2024 11:36:52 +0100 Subject: [PATCH 08/11] f --- dpytools/logging/logger.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/dpytools/logging/logger.py b/dpytools/logging/logger.py index 29384a4..5218aa1 100644 --- a/dpytools/logging/logger.py +++ b/dpytools/logging/logger.py @@ -73,24 +73,13 @@ def _log( error: Optional[List] = None, data: Optional[Dict] = None, raw: str = None, - response = Optional[requests.Response], + response : Optional[requests.Response] = None, ): data_dict = data if data is not None else {} data_dict["level"] = logging.getLevelName(level) # match dp logging structue # https://github.com/ONSdigital/dp-standards/blob/main/LOGGING_STANDARDS.md - log_event = { - "severity": level_to_severity(level), - "event": event, - "created_at": datetime.now(timezone.utc).isoformat(), - "namespace": self.namespace, - "trace_id": "not-implemented", - "span_id": "not-implemented", - "data": data_dict, - "raw": raw, - "errors": create_error_dict(error) if error is not None else None, - } r_dict = { "method": response.request.method, @@ -107,6 +96,19 @@ def _log( reponse_dict = r_dict if response is not None else {} + log_event = { + "severity": level_to_severity(level), + "event": event, + "created_at": datetime.now(timezone.utc).isoformat(), + "namespace": self.namespace, + "trace_id": "not-implemented", + "span_id": "not-implemented", + "data": data_dict, + "response_dict" : reponse_dict, + "raw": raw, + "errors": create_error_dict(error) if error is not None else None, + } + self._logger.log(**log_event) if self.flush_stdout_after_log_entry is True: From c4fbc6148e8a92d6cbdfb5e311b916e67fb22379 Mon Sep 17 00:00:00 2001 From: Jozsef K Date: Thu, 12 Sep 2024 11:41:10 +0100 Subject: [PATCH 09/11] f --- dpytools/logging/logger.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/dpytools/logging/logger.py b/dpytools/logging/logger.py index 5218aa1..8a61f57 100644 --- a/dpytools/logging/logger.py +++ b/dpytools/logging/logger.py @@ -81,20 +81,23 @@ def _log( # match dp logging structue # https://github.com/ONSdigital/dp-standards/blob/main/LOGGING_STANDARDS.md - r_dict = { - "method": response.request.method, - "scheme": get_scheme(response.url), - "host": get_domain(response.url), - "port": get_port(response.url), - "path": response.request.path_url, - "status_code" : response.status_code, - "started_at":get_start_date(response.headers["Date"]), - "ended_at":get_end_date(response.elapsed, response.headers["Date"]), - "duration": calculate_duration_in_nanoseconds(response.elapsed, response.headers["Date"]), - "response_content_length":len(response.content) - } - - reponse_dict = r_dict if response is not None else {} + if response is not None: + r_dict = { + "method": response.request.method, + "scheme": get_scheme(response.url), + "host": get_domain(response.url), + "port": get_port(response.url), + "path": response.request.path_url, + "status_code" : response.status_code, + "started_at":get_start_date(response.headers["Date"]), + "ended_at":get_end_date(response.elapsed, response.headers["Date"]), + "duration": calculate_duration_in_nanoseconds(response.elapsed, response.headers["Date"]), + "response_content_length":len(response.content) + } + else: + r_dict = None + + #reponse_dict = r_dict if response is not None else {} log_event = { "severity": level_to_severity(level), @@ -104,7 +107,7 @@ def _log( "trace_id": "not-implemented", "span_id": "not-implemented", "data": data_dict, - "response_dict" : reponse_dict, + "response_dict" : r_dict, "raw": raw, "errors": create_error_dict(error) if error is not None else None, } From 5b7199ed6ec969ea26341f250f41a4d261c24488 Mon Sep 17 00:00:00 2001 From: Jozsef K Date: Thu, 12 Sep 2024 11:42:01 +0100 Subject: [PATCH 10/11] fix --- dpytools/logging/logger.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/dpytools/logging/logger.py b/dpytools/logging/logger.py index 8a61f57..268166c 100644 --- a/dpytools/logging/logger.py +++ b/dpytools/logging/logger.py @@ -96,8 +96,6 @@ def _log( } else: r_dict = None - - #reponse_dict = r_dict if response is not None else {} log_event = { "severity": level_to_severity(level), From 27838df193858e68448cf9daa0275813f4acf989 Mon Sep 17 00:00:00 2001 From: Jozsef K Date: Fri, 13 Sep 2024 11:44:08 +0100 Subject: [PATCH 11/11] fix --- dpytools/logging/utility.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dpytools/logging/utility.py b/dpytools/logging/utility.py index 4602db4..6603a80 100644 --- a/dpytools/logging/utility.py +++ b/dpytools/logging/utility.py @@ -63,7 +63,7 @@ def get_scheme(url: str) -> str: return scheme def get_domain(url: str) -> str: - """This funciton will return the domain name from the provided url.""" + """This function will return the domain name from the provided url.""" #Parsing url to extract the domain name parsed_url = urlparse(url) @@ -72,11 +72,11 @@ def get_domain(url: str) -> str: return domain def get_port(url: str) ->int: - """This funciton will return the port number form the provided url.""" + """This function will return the port number form the provided url.""" #Parsing url parsed_url = urlparse(url) - #checking if the port was give if not scheking scheme for port number + #checking if the port was give if not checking scheme for port number if parsed_url.port is None: if parsed_url.scheme == "http": return 80 @@ -92,7 +92,7 @@ def get_start_date(date: str) ->str: return strp_time.isoformat() + "Z" def get_end_date(time_delta: timedelta, date: str) ->str: - """This function will calculate the end_date by addint the duration to the start date.""" + """This function will calculate the end_date by adding the duration to the start date.""" strp_time = datetime.strptime(date, '%a, %d %b %Y %H:%M:%S GMT') td = timedelta(microseconds=time_delta.microseconds) @@ -102,7 +102,7 @@ def get_end_date(time_delta: timedelta, date: str) ->str: return end_date.isoformat() + "Z" def calculate_duration_in_nanoseconds(time_delta: timedelta, date: str)->int: - """This funciton will convert the duration from Miliseconds to Nanoseconds.""" + """This function will convert the duration from Miliseconds to Nanoseconds.""" strp_time = datetime.strptime(date, '%a, %d %b %Y %H:%M:%S GMT') td = timedelta(microseconds=time_delta.microseconds) @@ -115,7 +115,7 @@ def calculate_duration_in_nanoseconds(time_delta: timedelta, date: str)->int: def get_content_length(res: Response)->int: """ - This funciton will try to get the 'Content-Lenght' + This function will try to get the 'Content-Lenght' if there is noone it will return a default 0. """ try: