From ce20b6c0db95d0bdccdef1777853dbae0e59403e Mon Sep 17 00:00:00 2001 From: sgoral-splunk <138458044+sgoral-splunk@users.noreply.github.com> Date: Wed, 26 Jun 2024 13:00:24 +0200 Subject: [PATCH 1/6] chore: merge main into develop (#375) merge main into develop --------- Co-authored-by: srv-rr-github-token <94607705+srv-rr-github-token@users.noreply.github.com> --- .releaserc | 2 +- pyproject.toml | 2 +- solnlib/__init__.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.releaserc b/.releaserc index e936f184..dce467f0 100644 --- a/.releaserc +++ b/.releaserc @@ -55,7 +55,7 @@ "@semantic-release/git", { "assets": ["NOTICE", "pyproject.toml", "solnlib/__init__.py"], - "message": "chore(release): ${nextRelease.version}\n\n${nextRelease.notes} [ci skip]", + "message": "chore(release): ${nextRelease.version}\n\n${nextRelease.notes}", }, ], ["@semantic-release/github", { "assets": ["NOTICE", "pyproject.toml"] }], diff --git a/pyproject.toml b/pyproject.toml index 2d61b050..1e98b031 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ [tool.poetry] name = "solnlib" -version = "5.1.0-beta.2" +version = "5.1.0" description = "The Splunk Software Development Kit for Splunk Solutions" authors = ["Splunk "] license = "Apache-2.0" diff --git a/solnlib/__init__.py b/solnlib/__init__.py index 32a94f26..0fa8a8e6 100644 --- a/solnlib/__init__.py +++ b/solnlib/__init__.py @@ -56,4 +56,4 @@ "utils", ] -__version__ = "5.1.0-beta.2" +__version__ = "5.1.0" From e126f1a8514b0864838bc83b54a00232d3e25d5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20K=C4=99dziak?= Date: Tue, 23 Jul 2024 10:15:42 +0200 Subject: [PATCH 2/6] feat: add log_level attribute (#382) This PR adds a new event field to logs (log_level). Example log before this change: ``` 2024-07-15 10:01:22,969 ERROR pid=49789 tid=MainThread file=helper_one.py:stream_events:19 | Test log ``` Example log after this change: ``` 2024-07-15 10:01:22,969 log_level=ERROR pid=49789 tid=MainThread file=helper_one.py:stream_events:19 | Test log ``` --- solnlib/log.py | 2 +- tests/unit/test_log.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/solnlib/log.py b/solnlib/log.py index 3f1958e1..05815059 100644 --- a/solnlib/log.py +++ b/solnlib/log.py @@ -79,7 +79,7 @@ class Logs(metaclass=Singleton): _default_directory = None _default_namespace = None _default_log_format = ( - "%(asctime)s %(levelname)s pid=%(process)d tid=%(threadName)s " + "%(asctime)s log_level=%(levelname)s pid=%(process)d tid=%(threadName)s " "file=%(filename)s:%(funcName)s:%(lineno)d | %(message)s" ) _default_log_level = logging.INFO diff --git a/tests/unit/test_log.py b/tests/unit/test_log.py index aa0298d4..b29029b4 100644 --- a/tests/unit/test_log.py +++ b/tests/unit/test_log.py @@ -18,10 +18,13 @@ import json import multiprocessing import os +import re import shutil import threading import traceback import time +from textwrap import dedent + import pytest from unittest import mock @@ -300,3 +303,33 @@ class AddonComplexError(Exception): mock_logger.log.assert_called_with( logging.ERROR, f"exc_l={result} \n{traceback.format_exc()}\n" ) + + +def test_log_format(monkeypatch, tmp_path): + log_file = tmp_path / "logging_levels.log" + + monkeypatch.setattr(log.Logs, "_get_log_file", lambda _, name: str(log_file)) + + logger = log.Logs().get_logger("logging_levels") + + logger.warning("log 2") + logger.error("log 3") + + log_content = transform_log(log_file.read_text()) + + assert ( + log_content + == dedent( + """ + 2024-03-23 10:15:20,555 log_level=WARNING pid=1234 tid=MainThread file=test_file.py:test_func:123 | log 2 + 2024-03-23 10:15:20,555 log_level=ERROR pid=1234 tid=MainThread file=test_file.py:test_func:123 | log 3 + """, + ).lstrip() + ) + + +def transform_log(log: str) -> str: + log = re.sub(r"pid=\d+", "pid=1234", log) + log = re.sub(r"file=[^ ]+", "file=test_file.py:test_func:123", log) + log = re.sub(r"\d{4}-\d\d-\d\d \d\d[^ ]+", "2024-03-23 10:15:20,555", log) + return log From 90bb44776d99a453ff2256ac885e6a1c3c6ad73e Mon Sep 17 00:00:00 2001 From: srv-rr-github-token <94607705+srv-rr-github-token@users.noreply.github.com> Date: Tue, 23 Jul 2024 08:20:56 +0000 Subject: [PATCH 3/6] chore(release): 5.1.0-beta.3 # [5.1.0-beta.3](https://github.com/splunk/addonfactory-solutions-library-python/compare/v5.1.0-beta.2...v5.1.0-beta.3) (2024-07-23) ### Features * add log_level attribute ([#382](https://github.com/splunk/addonfactory-solutions-library-python/issues/382)) ([e126f1a](https://github.com/splunk/addonfactory-solutions-library-python/commit/e126f1a8514b0864838bc83b54a00232d3e25d5b)) --- pyproject.toml | 2 +- solnlib/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1e98b031..de2e7cc4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ [tool.poetry] name = "solnlib" -version = "5.1.0" +version = "5.1.0-beta.3" description = "The Splunk Software Development Kit for Splunk Solutions" authors = ["Splunk "] license = "Apache-2.0" diff --git a/solnlib/__init__.py b/solnlib/__init__.py index 0fa8a8e6..99411772 100644 --- a/solnlib/__init__.py +++ b/solnlib/__init__.py @@ -56,4 +56,4 @@ "utils", ] -__version__ = "5.1.0" +__version__ = "5.1.0-beta.3" From cb5f3147ac8f8f9a6891d4fb6973f5e961acf30f Mon Sep 17 00:00:00 2001 From: sgoral-splunk <138458044+sgoral-splunk@users.noreply.github.com> Date: Fri, 9 Aug 2024 13:40:03 +0200 Subject: [PATCH 4/6] fix: add support of custom source in license usage (#383) **Issue number: [ADDON-72793](https://splunk.atlassian.net/browse/ADDON-72793)** ## Summary Added support for custom source in license usage ### Changes function `events_ingested` has new optional parameter `license_usage_source`. If provided this value will be save as `modular_input_name`. ## Checklist * [x] I have performed a self-review of this change * [x] Changes have been tested * [x] Changes are documented * [x] PR title follows [conventional commit semantics](https://www.conventionalcommits.org/en/v1.0.0/) --- solnlib/log.py | 6 ++- .../integration/test_bulletin_rest_client.py | 15 ++++++- tests/unit/test_log.py | 42 +++++++++++++++++-- 3 files changed, 58 insertions(+), 5 deletions(-) diff --git a/solnlib/log.py b/solnlib/log.py index 05815059..7c5cf11a 100644 --- a/solnlib/log.py +++ b/solnlib/log.py @@ -285,6 +285,7 @@ def events_ingested( index: str, account: str = None, host: str = None, + license_usage_source: str = None, ): """Specific function to log the basic information of events ingested for the monitoring dashboard. @@ -296,6 +297,7 @@ def events_ingested( sourcetype: Source type used to write event. n_events: Number of ingested events. index: Index used to write event. + license_usage_source: source used to match data with license_usage.log. account: Account used to write event. (optional) host: Host used to write event. (optional) """ @@ -310,7 +312,9 @@ def events_ingested( result = { "action": "events_ingested", - "modular_input_name": modular_input_name, + "modular_input_name": license_usage_source + if license_usage_source + else modular_input_name, "sourcetype_ingested": sourcetype, "n_events": n_events, "event_input": input_name, diff --git a/tests/integration/test_bulletin_rest_client.py b/tests/integration/test_bulletin_rest_client.py index 21e344b7..283e9c8b 100644 --- a/tests/integration/test_bulletin_rest_client.py +++ b/tests/integration/test_bulletin_rest_client.py @@ -56,9 +56,12 @@ def test_bulletin_rest_api(): bulletin_client_1 = _build_bulletin_manager("msg_name_1", session_key) bulletin_client_2 = _build_bulletin_manager("msg_name_2", session_key) + # clear bulletin before tests + _clear_bulletin() + bulletin_client_1.create_message( "new message to bulletin", - capabilities=["apps_restore", "delete_messages"], + capabilities=["apps_restore", "edit_roles"], roles=["admin"], ) @@ -106,3 +109,13 @@ def test_bulletin_rest_api(): get_all_msg = bulletin_client_1.get_all_messages() assert len(get_all_msg["entry"]) == 0 + + +def _clear_bulletin(): + session_key = context.get_session_key() + bulletin_client = _build_bulletin_manager("", session_key) + + msg_to_del = [el["name"] for el in bulletin_client.get_all_messages()["entry"]] + for msg in msg_to_del: + endpoint = f"{bulletin_client.MESSAGES_ENDPOINT}/{msg}" + bulletin_client._rest_client.delete(endpoint) diff --git a/tests/unit/test_log.py b/tests/unit/test_log.py index b29029b4..f3688e74 100644 --- a/tests/unit/test_log.py +++ b/tests/unit/test_log.py @@ -243,6 +243,42 @@ def test_events_ingested_invalid_input(): assert exp_msg == str(excinfo.value) +def test_events_ingested_custom_license_usage(): + with mock.patch("logging.Logger") as mock_logger: + log.events_ingested( + mock_logger, + "input_type://input_name", + "sourcetype", + 5, + "default", + license_usage_source="custom:license:source", + ) + + mock_logger.log.assert_called_once_with( + logging.INFO, + "action=events_ingested modular_input_name=custom:license:source sourcetype_ingested=sourcetype " + "n_events=5 event_input=input_name event_index=default", + ) + + with mock.patch("logging.Logger") as mock_logger: + log.events_ingested( + mock_logger, + "demo://modular_input_name", + "sourcetype", + 5, + "default", + host="abcd", + account="test_acc", + license_usage_source="custom:license:source:123", + ) + + mock_logger.log.assert_called_once_with( + logging.INFO, + "action=events_ingested modular_input_name=custom:license:source:123 sourcetype_ingested=sourcetype n_" + "events=5 event_input=modular_input_name event_index=default event_account=test_acc event_host=abcd", + ) + + def test_log_exceptions_full_msg(): start_msg = "some msg before exception" with mock.patch("logging.Logger") as mock_logger: @@ -321,9 +357,9 @@ def test_log_format(monkeypatch, tmp_path): log_content == dedent( """ - 2024-03-23 10:15:20,555 log_level=WARNING pid=1234 tid=MainThread file=test_file.py:test_func:123 | log 2 - 2024-03-23 10:15:20,555 log_level=ERROR pid=1234 tid=MainThread file=test_file.py:test_func:123 | log 3 - """, + 2024-03-23 10:15:20,555 log_level=WARNING pid=1234 tid=MainThread file=test_file.py:test_func:123 | log 2 + 2024-03-23 10:15:20,555 log_level=ERROR pid=1234 tid=MainThread file=test_file.py:test_func:123 | log 3 + """, ).lstrip() ) From 5e2269de4f699e16757642915ad693b0e7d39106 Mon Sep 17 00:00:00 2001 From: srv-rr-github-token <94607705+srv-rr-github-token@users.noreply.github.com> Date: Fri, 9 Aug 2024 11:46:04 +0000 Subject: [PATCH 5/6] chore(release): 5.1.0-beta.4 # [5.1.0-beta.4](https://github.com/splunk/addonfactory-solutions-library-python/compare/v5.1.0-beta.3...v5.1.0-beta.4) (2024-08-09) ### Bug Fixes * add support of custom source in license usage ([#383](https://github.com/splunk/addonfactory-solutions-library-python/issues/383)) ([cb5f314](https://github.com/splunk/addonfactory-solutions-library-python/commit/cb5f3147ac8f8f9a6891d4fb6973f5e961acf30f)) --- pyproject.toml | 2 +- solnlib/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index de2e7cc4..2421076a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ [tool.poetry] name = "solnlib" -version = "5.1.0-beta.3" +version = "5.1.0-beta.4" description = "The Splunk Software Development Kit for Splunk Solutions" authors = ["Splunk "] license = "Apache-2.0" diff --git a/solnlib/__init__.py b/solnlib/__init__.py index 99411772..5efbab61 100644 --- a/solnlib/__init__.py +++ b/solnlib/__init__.py @@ -56,4 +56,4 @@ "utils", ] -__version__ = "5.1.0-beta.3" +__version__ = "5.1.0-beta.4" From f02c2a35bc8bf6eaa24984c35e68782d41faec4d Mon Sep 17 00:00:00 2001 From: srv-rr-github-token <94607705+srv-rr-github-token@users.noreply.github.com> Date: Tue, 13 Aug 2024 09:22:31 +0000 Subject: [PATCH 6/6] chore(release): 5.2.0-beta.1 # [5.2.0-beta.1](https://github.com/splunk/addonfactory-solutions-library-python/compare/v5.1.2...v5.2.0-beta.1) (2024-08-13) ### Bug Fixes * add support of custom source in license usage ([#383](https://github.com/splunk/addonfactory-solutions-library-python/issues/383)) ([cb5f314](https://github.com/splunk/addonfactory-solutions-library-python/commit/cb5f3147ac8f8f9a6891d4fb6973f5e961acf30f)) ### Features * add log_level attribute ([#382](https://github.com/splunk/addonfactory-solutions-library-python/issues/382)) ([e126f1a](https://github.com/splunk/addonfactory-solutions-library-python/commit/e126f1a8514b0864838bc83b54a00232d3e25d5b)) --- pyproject.toml | 2 +- solnlib/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 92307a7c..76add752 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,7 +16,7 @@ [tool.poetry] name = "solnlib" -version = "5.1.2" +version = "5.2.0-beta.1" description = "The Splunk Software Development Kit for Splunk Solutions" authors = ["Splunk "] license = "Apache-2.0" diff --git a/solnlib/__init__.py b/solnlib/__init__.py index 50ae5eb9..425e6531 100644 --- a/solnlib/__init__.py +++ b/solnlib/__init__.py @@ -56,4 +56,4 @@ "utils", ] -__version__ = "5.1.2" +__version__ = "5.2.0-beta.1"