From 89a647a153efff80ef7ec0ef4035ec9fd9acfec5 Mon Sep 17 00:00:00 2001 From: znerol Date: Sun, 28 Nov 2021 17:58:11 +0100 Subject: [PATCH 01/12] feat(logging) Add a logging.Filter implementation --- README.md | 24 ++++++++++++++++++++++++ anonip.py | 23 +++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/README.md b/README.md index 81dfabf..84c22a4 100644 --- a/README.md +++ b/README.md @@ -149,6 +149,30 @@ for line in data: ``` + +### As a python logging.Filter + +```python +import logging + +from anonip import AnonipFilter + +if __name__ == '__main__': + handler = logging.StreamHandler() + handler.addFilter(AnonipFilter()) + logging.basicConfig( + level=logging.DEBUG, + format='%(asctime)s - %(levelname)s - %(message)s', + handlers=[handler] + ) + + logging.debug('192.0.2.123 - call from root logger') + + logger = logging.getLogger('child') + logger.info('2001:db8:abcd:ef01:2345:6789:abcd:ef01 - call from child logger') +``` + + ### Python 2 or 3? For compatibility reasons, anonip uses the shebang `#! /usr/bin/env python`. This will default to python2 on all Linux distributions except for Arch Linux. diff --git a/anonip.py b/anonip.py index 52ac7c2..b1b89a3 100755 --- a/anonip.py +++ b/anonip.py @@ -260,6 +260,29 @@ def truncate_address(self, ip): return ip.supernet(new_prefix=self._prefixes[ip.version])[0] +class AnonipFilter: + def __init__(self, keys=['msg'], anonip={}): + """ + An implementation of Python logging.Filter using anonip. + + :param keys: list of LogRecord attributes to filter. Defaults to ['msg'] + :param anonip: dict of parameters for Anonip instance + """ + self.keys = list(keys) + self.anonip = Anonip(**anonip) + + def filter(self, record: logging.LogRecord): + """ + See logging.Filter.filter() + """ + for key in self.keys: + if hasattr(record, key): + line = getattr(record, key) + setattr(record, key, self.anonip.process_line(line)) + + return True + + def _validate_ipmask(mask, bits=32): """ Verify if the supplied ip mask is valid. From 0b3609a292f7f3ffdeb617217607611577a8431f Mon Sep 17 00:00:00 2001 From: znerol Date: Mon, 29 Nov 2021 20:02:13 +0100 Subject: [PATCH 02/12] chore(ci): Trigger tests From 2823de59292dd23f51c54c578ef0cb8a11c560e2 Mon Sep 17 00:00:00 2001 From: znerol Date: Mon, 29 Nov 2021 22:23:37 +0100 Subject: [PATCH 03/12] fix(tests): Add test coverage for AnonipFilter --- anonip.py | 16 +++++++++---- tests.py | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 5 deletions(-) diff --git a/anonip.py b/anonip.py index b1b89a3..fe63d07 100755 --- a/anonip.py +++ b/anonip.py @@ -261,21 +261,27 @@ def truncate_address(self, ip): class AnonipFilter: - def __init__(self, keys=['msg'], anonip={}): + def __init__(self, args=None, extra=None, anonip=None): """ An implementation of Python logging.Filter using anonip. - :param keys: list of LogRecord attributes to filter. Defaults to ['msg'] + :param args: list of log message args to filter. Defaults to [] + :param extra: list of LogRecord attributes to filter. Defaults to ['msg'] :param anonip: dict of parameters for Anonip instance """ - self.keys = list(keys) - self.anonip = Anonip(**anonip) + self.args = args or [] + self.extra = extra or ['msg'] + self.anonip = Anonip(**(anonip or {})) def filter(self, record: logging.LogRecord): """ See logging.Filter.filter() """ - for key in self.keys: + for key in self.args: + if key in record.args: + record.args[key] = self.anonip.process_line(record.args[key]) + + for key in self.extra: if hasattr(record, key): line = getattr(record, key) setattr(record, key, self.anonip.process_line(line)) diff --git a/tests.py b/tests.py index 7762f0b..b62d161 100755 --- a/tests.py +++ b/tests.py @@ -319,3 +319,73 @@ def test_properties_columns(): assert a.columns == [0] a.columns = [5, 6] assert a.columns == [4, 5] + +def test_logging_filter_defaults(caplog): + logging.disable(logging.NOTSET) + logging.getLogger("anonip").setLevel(logging.CRITICAL) + + logger = logging.getLogger("filter_defaults") + logger.addFilter(anonip.AnonipFilter()) + logger.setLevel(logging.INFO) + + logger.info("192.168.100.200 string") + logger.info("1.2.3.4 string") + logger.info("2001:0db8:85a3:0000:0000:8a2e:0370:7334 string") + logger.info("2a00:1450:400a:803::200e string") + + assert caplog.record_tuples == [ + ("filter_defaults", logging.INFO, "192.168.96.0 string"), + ("filter_defaults", logging.INFO, "1.2.0.0 string"), + ("filter_defaults", logging.INFO, "2001:db8:85a0:: string"), + ("filter_defaults", logging.INFO, "2a00:1450:4000:: string"), + ] + + logging.disable(logging.CRITICAL) + +def test_logging_filter_args(caplog): + logging.disable(logging.NOTSET) + logging.getLogger("anonip").setLevel(logging.CRITICAL) + + logger = logging.getLogger("filter_args") + logger.addFilter(anonip.AnonipFilter(args=['ip', 'non-existing-attr'], extra=[])) + logger.setLevel(logging.INFO) + + logger.info("%(ip)s string", {"ip": "192.168.100.200"}) + logger.info("string %(ip)s", {"ip": "1.2.3.4"}) + logger.info("%(ip)s string", {"ip": "2001:0db8:85a3:0000:0000:8a2e:0370:7334"}) + logger.info("string") + + assert caplog.record_tuples == [ + ("filter_args", logging.INFO, "192.168.96.0 string"), + ("filter_args", logging.INFO, "string 1.2.0.0"), + ("filter_args", logging.INFO, "2001:db8:85a0:: string"), + ("filter_args", logging.INFO, "string"), + ] + + logging.disable(logging.CRITICAL) + +def test_logging_filter_extra(caplog): + logging.disable(logging.NOTSET) + logging.getLogger("anonip").setLevel(logging.CRITICAL) + + logger = logging.getLogger("filter_args") + logger.addFilter(anonip.AnonipFilter(extra=['ip', 'non-existing-key'], anonip={"ipv4mask": 16, "ipv6mask": 64})) + logger.setLevel(logging.INFO) + + logger.info("string", extra={"ip": "192.168.100.200"}) + logger.info("string", extra={"ip": "1.2.3.4"}) + logger.info("string", extra={"ip": "2001:0db8:85a3:0000:0000:8a2e:0370:7334"}) + logger.info("string", extra={"ip": "2a00:1450:400a:803::200e"}) + + expected = [ + "192.168.0.0", + "1.2.0.0", + "2001:db8:85a3::", + "2a00:1450:400a:803::", + ] + + actual = [record.ip for record in caplog.records] + + assert actual == expected + + logging.disable(logging.CRITICAL) From 63ca5f19864973ff829fc3a7b6a6bb0597217a6b Mon Sep 17 00:00:00 2001 From: znerol Date: Mon, 29 Nov 2021 22:26:11 +0100 Subject: [PATCH 04/12] fix(logging): Python 2 compatibility --- anonip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/anonip.py b/anonip.py index fe63d07..27e03f5 100755 --- a/anonip.py +++ b/anonip.py @@ -273,7 +273,7 @@ def __init__(self, args=None, extra=None, anonip=None): self.extra = extra or ['msg'] self.anonip = Anonip(**(anonip or {})) - def filter(self, record: logging.LogRecord): + def filter(self, record): """ See logging.Filter.filter() """ From f456c5726e45f30d24a269067b1eddd19e3f6a32 Mon Sep 17 00:00:00 2001 From: znerol Date: Mon, 29 Nov 2021 22:28:11 +0100 Subject: [PATCH 05/12] fix(style): Coding style --- anonip.py | 4 ++-- tests.py | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/anonip.py b/anonip.py index 27e03f5..5983669 100755 --- a/anonip.py +++ b/anonip.py @@ -266,11 +266,11 @@ def __init__(self, args=None, extra=None, anonip=None): An implementation of Python logging.Filter using anonip. :param args: list of log message args to filter. Defaults to [] - :param extra: list of LogRecord attributes to filter. Defaults to ['msg'] + :param extra: list of LogRecord attributes to filter. Defaults to ["msg"] :param anonip: dict of parameters for Anonip instance """ self.args = args or [] - self.extra = extra or ['msg'] + self.extra = extra or ["msg"] self.anonip = Anonip(**(anonip or {})) def filter(self, record): diff --git a/tests.py b/tests.py index b62d161..2eeee58 100755 --- a/tests.py +++ b/tests.py @@ -320,6 +320,7 @@ def test_properties_columns(): a.columns = [5, 6] assert a.columns == [4, 5] + def test_logging_filter_defaults(caplog): logging.disable(logging.NOTSET) logging.getLogger("anonip").setLevel(logging.CRITICAL) @@ -342,6 +343,7 @@ def test_logging_filter_defaults(caplog): logging.disable(logging.CRITICAL) + def test_logging_filter_args(caplog): logging.disable(logging.NOTSET) logging.getLogger("anonip").setLevel(logging.CRITICAL) @@ -364,6 +366,7 @@ def test_logging_filter_args(caplog): logging.disable(logging.CRITICAL) + def test_logging_filter_extra(caplog): logging.disable(logging.NOTSET) logging.getLogger("anonip").setLevel(logging.CRITICAL) From e630dd1e01f415828aaa8caa1fa27d5922066528 Mon Sep 17 00:00:00 2001 From: znerol Date: Mon, 29 Nov 2021 22:30:21 +0100 Subject: [PATCH 06/12] fix(tests): More coding style --- tests.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests.py b/tests.py index 2eeee58..1f1dc49 100755 --- a/tests.py +++ b/tests.py @@ -372,7 +372,11 @@ def test_logging_filter_extra(caplog): logging.getLogger("anonip").setLevel(logging.CRITICAL) logger = logging.getLogger("filter_args") - logger.addFilter(anonip.AnonipFilter(extra=['ip', 'non-existing-key'], anonip={"ipv4mask": 16, "ipv6mask": 64})) + logger.addFilter( + anonip.AnonipFilter( + extra=['ip', 'non-existing-key'], anonip={"ipv4mask": 16, "ipv6mask": 64} + ) + ) logger.setLevel(logging.INFO) logger.info("string", extra={"ip": "192.168.100.200"}) From fe9f79424daa23ed3fe1d9833a5bbdf0e3e721e7 Mon Sep 17 00:00:00 2001 From: znerol Date: Mon, 29 Nov 2021 22:32:12 +0100 Subject: [PATCH 07/12] fix(tests): Even more coding style --- tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests.py b/tests.py index 1f1dc49..ab9a599 100755 --- a/tests.py +++ b/tests.py @@ -349,7 +349,7 @@ def test_logging_filter_args(caplog): logging.getLogger("anonip").setLevel(logging.CRITICAL) logger = logging.getLogger("filter_args") - logger.addFilter(anonip.AnonipFilter(args=['ip', 'non-existing-attr'], extra=[])) + logger.addFilter(anonip.AnonipFilter(args=["ip", "non-existing-attr"], extra=[])) logger.setLevel(logging.INFO) logger.info("%(ip)s string", {"ip": "192.168.100.200"}) @@ -374,7 +374,7 @@ def test_logging_filter_extra(caplog): logger = logging.getLogger("filter_args") logger.addFilter( anonip.AnonipFilter( - extra=['ip', 'non-existing-key'], anonip={"ipv4mask": 16, "ipv6mask": 64} + extra=["ip", "non-existing-key"], anonip={"ipv4mask": 16, "ipv6mask": 64} ) ) logger.setLevel(logging.INFO) From f174aaec5788cc471c2db74ec1674edd42a58759 Mon Sep 17 00:00:00 2001 From: znerol Date: Sat, 4 Dec 2021 19:26:57 +0100 Subject: [PATCH 08/12] fix(logging): Skip own log records, skip non-string values --- anonip.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/anonip.py b/anonip.py index 5983669..95f3a0b 100755 --- a/anonip.py +++ b/anonip.py @@ -277,14 +277,16 @@ def filter(self, record): """ See logging.Filter.filter() """ - for key in self.args: - if key in record.args: - record.args[key] = self.anonip.process_line(record.args[key]) - - for key in self.extra: - if hasattr(record, key): - line = getattr(record, key) - setattr(record, key, self.anonip.process_line(line)) + if record.name != "anonip": + for key in self.args: + if key in record.args: + record.args[key] = self.anonip.process_line(record.args[key]) + + for key in self.extra: + if hasattr(record, key): + value = getattr(record, key) + if (isinstance(value, str)): + setattr(record, key, self.anonip.process_line(value)) return True From 3925a5d136809548176330dc3a669bde342af1f7 Mon Sep 17 00:00:00 2001 From: znerol Date: Sat, 4 Dec 2021 19:49:28 +0100 Subject: [PATCH 09/12] fix(logging): record.args is either a mapping or a sequence --- anonip.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/anonip.py b/anonip.py index 95f3a0b..271b2e9 100755 --- a/anonip.py +++ b/anonip.py @@ -43,6 +43,7 @@ import logging import sys from io import open +from collections import abc try: import ipaddress @@ -279,8 +280,14 @@ def filter(self, record): """ if record.name != "anonip": for key in self.args: - if key in record.args: - record.args[key] = self.anonip.process_line(record.args[key]) + if isinstance(record.args, abc.Mapping): + if key in record.args: + record.args[key] = self.anonip.process_line(record.args[key]) + elif isinstance(record.args, abc.Sequence): + if key < len(record.args): + # Convert tuple to list if necessary. + record.args = list(record.args) + record.args[key] = self.anonip.process_line(record.args[key]) for key in self.extra: if hasattr(record, key): From 2a8271d3c35b614fa7be474e3c165a1c079acb45 Mon Sep 17 00:00:00 2001 From: znerol Date: Sat, 4 Dec 2021 20:03:35 +0100 Subject: [PATCH 10/12] fix(logging): Be more defensive when mutating record.args --- anonip.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/anonip.py b/anonip.py index 271b2e9..0db4cca 100755 --- a/anonip.py +++ b/anonip.py @@ -282,12 +282,19 @@ def filter(self, record): for key in self.args: if isinstance(record.args, abc.Mapping): if key in record.args: - record.args[key] = self.anonip.process_line(record.args[key]) + value = record.args[key] + if isinstance(value, str): + record.args[key] = self.anonip.process_line(value) elif isinstance(record.args, abc.Sequence): if key < len(record.args): - # Convert tuple to list if necessary. - record.args = list(record.args) - record.args[key] = self.anonip.process_line(record.args[key]) + value = record.args[key] + if isinstance(value, str): + is_tuple = isinstance(record.args, tuple) + if is_tuple: + record.args = list(record.args) + record.args[key] = self.anonip.process_line(value) + if is_tuple: + record.args = tuple(record.args) for key in self.extra: if hasattr(record, key): From 48da166fe7f858ad1fb366d3c989b0b1795540f5 Mon Sep 17 00:00:00 2001 From: znerol Date: Sat, 4 Dec 2021 20:12:06 +0100 Subject: [PATCH 11/12] fix(logging): Correct defaults --- anonip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/anonip.py b/anonip.py index 0db4cca..0e8ef1c 100755 --- a/anonip.py +++ b/anonip.py @@ -270,8 +270,8 @@ def __init__(self, args=None, extra=None, anonip=None): :param extra: list of LogRecord attributes to filter. Defaults to ["msg"] :param anonip: dict of parameters for Anonip instance """ - self.args = args or [] - self.extra = extra or ["msg"] + self.args = [] if args is None else args + self.extra = ["msg"] if extra is None else extra self.anonip = Anonip(**(anonip or {})) def filter(self, record): From 6661fa21b69403ffe7e3baf5d577e54b6db2866b Mon Sep 17 00:00:00 2001 From: znerol Date: Sat, 4 Dec 2021 20:18:55 +0100 Subject: [PATCH 12/12] fix(logging): No defaults --- anonip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/anonip.py b/anonip.py index 0e8ef1c..3fd7808 100755 --- a/anonip.py +++ b/anonip.py @@ -267,11 +267,11 @@ def __init__(self, args=None, extra=None, anonip=None): An implementation of Python logging.Filter using anonip. :param args: list of log message args to filter. Defaults to [] - :param extra: list of LogRecord attributes to filter. Defaults to ["msg"] + :param extra: list of LogRecord attributes to filter. Defaults to [] :param anonip: dict of parameters for Anonip instance """ self.args = [] if args is None else args - self.extra = ["msg"] if extra is None else extra + self.extra = [] if extra is None else extra self.anonip = Anonip(**(anonip or {})) def filter(self, record):