From 0a173759fa8bdd8df67b753a0c9589a6cea8d796 Mon Sep 17 00:00:00 2001 From: hetangmodi-crest Date: Mon, 6 May 2024 10:50:47 +0530 Subject: [PATCH 01/10] feat(generation): add logic to identify files created by UCC --- .../auto_gen_comparator.py | 53 +++++++++++++++++++ splunk_add_on_ucc_framework/commands/build.py | 15 +++++- .../alert_actions_conf_gen.py | 7 ++- .../alert_actions_py_gen.py | 8 ++- .../commands/modular_alert_builder/builder.py | 10 ++-- 5 files changed, 84 insertions(+), 9 deletions(-) create mode 100644 splunk_add_on_ucc_framework/auto_gen_comparator.py diff --git a/splunk_add_on_ucc_framework/auto_gen_comparator.py b/splunk_add_on_ucc_framework/auto_gen_comparator.py new file mode 100644 index 000000000..48908c60f --- /dev/null +++ b/splunk_add_on_ucc_framework/auto_gen_comparator.py @@ -0,0 +1,53 @@ +from os import walk +from os.path import sep +from typing import List, Dict +from logging import Logger + + +class CodeGeneratorDiffChecker: + def __init__(self, src_dir: str, dst_dir: str) -> None: + self.source_directory = src_dir + self.target_directory = dst_dir + self.common_files: Dict[str, str] = {} + + def find_common_files(self, ingore_file_list: List[str] = []) -> None: + # we add these two files as they are required to be present in source code + ingore_file_list.extend(["app.manifest", "README.txt"]) + + src_all_files = {} + for root, _, files in walk(self.source_directory): + for file in files: + src_all_files[file] = sep.join([root, file]) + + dest_all_files = {} + for root, _, files in walk(self.target_directory): + for file in files: + dest_all_files[file] = sep.join([root, file]) + dest_all_files["default.meta"] = sep.join([self.target_directory, "metadata"]) + + for file_name, file_path in dest_all_files.items(): + if file_name in src_all_files.keys(): + if file_name in ingore_file_list: + continue + self.common_files[file_name] = src_all_files[file_name] + + def print_files(self, logger: Logger) -> None: + if not self.common_files: + # nothing to print if there are no common files + return + + messages: List[str] = [] + messages.append("-" * 120) + messages.append( + "Below are the file(s) that are auto generated by the UCC framework. " + "The below files can be removed from your repository:" + ) + messages.extend( + [f"{idx}) {f}" for idx, f in enumerate(self.common_files.values())] + ) + messages.append( + "Please refer UCC framework documentation for the latest " + "features that ables you to remove the above files." + ) + messages.append("-" * 120) + logger.warning("\n".join(messages)) diff --git a/splunk_add_on_ucc_framework/commands/build.py b/splunk_add_on_ucc_framework/commands/build.py index 671c82f98..f8ec5f24f 100644 --- a/splunk_add_on_ucc_framework/commands/build.py +++ b/splunk_add_on_ucc_framework/commands/build.py @@ -36,6 +36,7 @@ utils, ) from splunk_add_on_ucc_framework import dashboard +from splunk_add_on_ucc_framework.auto_gen_comparator import CodeGeneratorDiffChecker from splunk_add_on_ucc_framework import app_conf as app_conf_lib from splunk_add_on_ucc_framework import meta_conf as meta_conf_lib from splunk_add_on_ucc_framework import server_conf as server_conf_lib @@ -523,6 +524,7 @@ def generate( app_manifest = _get_app_manifest(source) ta_name = app_manifest.get_addon_name() + auto_gen_ignore_list = [] gc_path = _get_and_check_global_config_path(source, config_path) if gc_path: logger.info(f"Using globalConfig file located @ {gc_path}") @@ -601,8 +603,10 @@ def generate( _add_modular_input(ta_name, global_config, output_directory) if global_config.has_alerts(): logger.info("Generating alerts code") - alert_builder.generate_alerts( - global_config, ta_name, internal_root_dir, output_directory + auto_gen_ignore_list.extend( + alert_builder.generate_alerts( + global_config, ta_name, internal_root_dir, output_directory + ) ) conf_file_names = [] @@ -667,6 +671,13 @@ def generate( removed_list = _remove_listed_files(ignore_list) if removed_list: logger.info("Removed:\n{}".format("\n".join(removed_list))) + + comparator = CodeGeneratorDiffChecker( + source, os.path.join(output_directory, ta_name) + ) + comparator.find_common_files(auto_gen_ignore_list) + comparator.print_files(logger) + utils.recursive_overwrite(source, os.path.join(output_directory, ta_name)) logger.info("Copied package directory") diff --git a/splunk_add_on_ucc_framework/commands/modular_alert_builder/alert_actions_conf_gen.py b/splunk_add_on_ucc_framework/commands/modular_alert_builder/alert_actions_conf_gen.py index 790b66c6f..a8629315e 100644 --- a/splunk_add_on_ucc_framework/commands/modular_alert_builder/alert_actions_conf_gen.py +++ b/splunk_add_on_ucc_framework/commands/modular_alert_builder/alert_actions_conf_gen.py @@ -17,7 +17,7 @@ import logging from os import linesep, makedirs, path as op import shutil -from typing import Dict, Any +from typing import Dict, Any, List from jinja2 import Environment, FileSystemLoader @@ -64,6 +64,7 @@ def __init__( "payload_format": "json", "icon_path": "alerticon.png", } + self.alerts_icon_list: List[str] = [] def get_local_conf_file_path(self, conf_name: str) -> str: local_path = op.join(self._package_path, "default") @@ -114,6 +115,7 @@ def generate_conf(self) -> None: elif k == "alert_props": if alert.get("iconFileName", "alerticon.png") != "alerticon.png": alert["alert_props"]["icon_path"] = alert["iconFileName"] + self.alerts_icon_list.append(alert["iconFileName"]) else: # we copy UCC framework's alerticon.png only when a custom isn't provided shutil.copy( @@ -250,12 +252,13 @@ def generate_spec(self) -> None: + 'object="alert_actions.conf.spec", object_type="file"' ) - def handle(self) -> None: + def handle(self) -> List[str]: self.add_default_settings() self.generate_conf() self.generate_spec() self.generate_eventtypes() self.generate_tags() + return self.alerts_icon_list def add_default_settings(self) -> None: for alert in self._alert_settings: diff --git a/splunk_add_on_ucc_framework/commands/modular_alert_builder/alert_actions_py_gen.py b/splunk_add_on_ucc_framework/commands/modular_alert_builder/alert_actions_py_gen.py index 30a4a31a2..4ca17e247 100644 --- a/splunk_add_on_ucc_framework/commands/modular_alert_builder/alert_actions_py_gen.py +++ b/splunk_add_on_ucc_framework/commands/modular_alert_builder/alert_actions_py_gen.py @@ -16,7 +16,7 @@ import logging import re from os import path as op -from typing import Any, Dict +from typing import Any, Dict, List from jinja2 import Environment, FileSystemLoader @@ -57,6 +57,7 @@ def __init__( lstrip_blocks=True, keep_trailing_newline=True, ) + self.alerts_script_list: List[str] = [] def _get_alert_py_name(self, alert: Any) -> str: return alert[ac.SHORT_NAME] + ".py" @@ -106,10 +107,13 @@ def gen_helper_py_file(self, alert: Any) -> None: rendered_content, ) - def handle(self) -> None: + def handle(self) -> List[str]: for alert in self._alert_actions_setting: logger.info( f"Generating Python file for alert action {alert[ac.SHORT_NAME]}" ) self.gen_main_py_file(alert) self.gen_helper_py_file(alert) + if alert.get("customScript"): + self.alerts_script_list.append(alert["customScript"]) + return self.alerts_script_list diff --git a/splunk_add_on_ucc_framework/commands/modular_alert_builder/builder.py b/splunk_add_on_ucc_framework/commands/modular_alert_builder/builder.py index 6f5dd9c04..566b32afd 100644 --- a/splunk_add_on_ucc_framework/commands/modular_alert_builder/builder.py +++ b/splunk_add_on_ucc_framework/commands/modular_alert_builder/builder.py @@ -27,6 +27,7 @@ from splunk_add_on_ucc_framework.commands.modular_alert_builder import ( alert_actions_py_gen, ) +from typing import List logger = logging.getLogger("ucc_gen") @@ -36,7 +37,8 @@ def generate_alerts( addon_name: str, internal_source_dir: str, output_dir: str, -) -> None: +) -> List[str]: + custom_file_list = [] envs = normalize.normalize( global_config.alerts, global_config.namespace, @@ -49,7 +51,7 @@ def generate_alerts( package_path=package_dir, internal_source_path=internal_source_dir, ) - conf_gen.handle() + custom_file_list.extend(conf_gen.handle()) html_gen = alert_actions_html_gen.AlertHtmlGenerator( input_setting=schema_content, @@ -62,4 +64,6 @@ def generate_alerts( input_setting=schema_content, package_path=package_dir, ) - py_gen.handle() + custom_file_list.extend(py_gen.handle()) + + return custom_file_list From bc53688e2ef05aa13231280270ab150c9f8760bd Mon Sep 17 00:00:00 2001 From: hetangmodi-crest Date: Mon, 6 May 2024 10:51:11 +0530 Subject: [PATCH 02/10] test(smoke): update smoke test with the expected logs --- tests/smoke/test_ucc_build.py | 76 ++++++++++++++++++++++++++++++++++- 1 file changed, 75 insertions(+), 1 deletion(-) diff --git a/tests/smoke/test_ucc_build.py b/tests/smoke/test_ucc_build.py index 756f78abf..5e561ed4a 100644 --- a/tests/smoke/test_ucc_build.py +++ b/tests/smoke/test_ucc_build.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Dict, Any +import re from tests.smoke import helpers from tests.unit import helpers as unit_helpers import addonfactory_splunk_conf_parser_lib as conf_parser @@ -40,7 +41,54 @@ def _compare_app_conf(expected_folder: str, actual_folder: str) -> None: assert expected_app_conf_dict == actual_app_conf_dict -def test_ucc_generate(): +def _compare_logging_tabs(package_dir: str, output_dir: str) -> None: + with open(Path(package_dir) / os.pardir / "globalConfig.json") as fp: + global_config = json.load(fp) + + with open( + Path(output_dir) / "appserver" / "static" / "js" / "build" / "globalConfig.json" + ) as fp: + static_config = json.load(fp) + + tab_exists = False + num = 0 + + for num, tab in enumerate(global_config["pages"]["configuration"]["tabs"]): + if tab.get("type", "") == "loggingTab": + tab_exists = True + break + + assert tab_exists + + static_tab = static_config["pages"]["configuration"]["tabs"][num] + + assert "type" not in static_tab + assert static_tab == { + "entity": [ + { + "defaultValue": "INFO", + "field": "loglevel", + "label": "Log level", + "options": { + "autoCompleteFields": [ + {"label": "DEBUG", "value": "DEBUG"}, + {"label": "INFO", "value": "INFO"}, + {"label": "WARNING", "value": "WARNING"}, + {"label": "ERROR", "value": "ERROR"}, + {"label": "CRITICAL", "value": "CRITICAL"}, + ], + "disableSearch": True, + }, + "type": "singleSelect", + "required": True, + } + ], + "name": "logging", + "title": "Logging", + } + + +def test_ucc_generate(caplog): package_folder = path.join( path.dirname(path.realpath(__file__)), "..", @@ -50,6 +98,19 @@ def test_ucc_generate(): "package", ) build.generate(source=package_folder) + for message in caplog.messages: + exp_msg = re.compile( + r".*(Below are the file\(s\) that are auto generated by the UCC framework\. The below files can be " + r"removed from your repository:.+Please refer UCC framework documentation for the latest features " + r"that ables you to remove the above files\.).*", + flags=re.DOTALL, + ) + if exp_msg.search(message): + # the warning log should be present due to a duplicate file already generated by UCC framework + assert True + break + else: + assert False def test_ucc_generate_with_config_param(): @@ -487,6 +548,19 @@ def test_ucc_generate_with_everything_uccignore(caplog): actual_file_path = path.join(actual_folder, *af) assert not path.exists(actual_file_path) + for message in caplog.messages: + exp_msg = re.compile( + r".*(Below are the file\(s\) that are auto generated by the UCC framework\. The below files can be " + r"removed from your repository:.+Please refer UCC framework documentation for the latest features " + r"that ables you to remove the above files\.).*", + flags=re.DOTALL, + ) + if exp_msg.search(message) is None: + # the warning log should be absent due to no duplicate files + assert True + else: + assert False + def test_ucc_generate_only_one_tab(): package_folder = path.join( From 6a17248c10fb53e5f1fcc6db0e9d7ea94cbaaa72 Mon Sep 17 00:00:00 2001 From: hetangmodi-crest Date: Tue, 7 May 2024 16:10:10 +0530 Subject: [PATCH 03/10] test(unit): add unit test case for source code --- .../auto_gen_comparator.py | 17 +++++- tests/unit/test_auto_gen_comparator.py | 60 +++++++++++++++++++ 2 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 tests/unit/test_auto_gen_comparator.py diff --git a/splunk_add_on_ucc_framework/auto_gen_comparator.py b/splunk_add_on_ucc_framework/auto_gen_comparator.py index 48908c60f..6c384c565 100644 --- a/splunk_add_on_ucc_framework/auto_gen_comparator.py +++ b/splunk_add_on_ucc_framework/auto_gen_comparator.py @@ -1,3 +1,18 @@ +# +# Copyright 2024 Splunk Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# from os import walk from os.path import sep from typing import List, Dict @@ -25,7 +40,7 @@ def find_common_files(self, ingore_file_list: List[str] = []) -> None: dest_all_files[file] = sep.join([root, file]) dest_all_files["default.meta"] = sep.join([self.target_directory, "metadata"]) - for file_name, file_path in dest_all_files.items(): + for file_name in dest_all_files.keys(): if file_name in src_all_files.keys(): if file_name in ingore_file_list: continue diff --git a/tests/unit/test_auto_gen_comparator.py b/tests/unit/test_auto_gen_comparator.py new file mode 100644 index 000000000..77af28c28 --- /dev/null +++ b/tests/unit/test_auto_gen_comparator.py @@ -0,0 +1,60 @@ +from splunk_add_on_ucc_framework import auto_gen_comparator +from unittest.mock import MagicMock + + +def test_print_files_blank(): + obj = auto_gen_comparator.CodeGeneratorDiffChecker("", "") + obj.common_files = {} + logger = MagicMock() + obj.print_files(logger) + + assert logger.warning.call_count == 0 + + +def test_print_files_not_blank(): + obj = auto_gen_comparator.CodeGeneratorDiffChecker("", "") + obj.common_files = {"file.1": "full_path_file.1"} + logger = MagicMock() + obj.print_files(logger) + + exp_msg = [] + exp_msg.append("-" * 120) + exp_msg.append( + "Below are the file(s) that are auto generated by the UCC framework. " + "The below files can be removed from your repository:" + ) + exp_msg.extend([f"{idx}) {f}" for idx, f in enumerate(obj.common_files.values())]) + exp_msg.append( + "Please refer UCC framework documentation for the latest " + "features that ables you to remove the above files." + ) + exp_msg.append("-" * 120) + + logger.warning.assert_called_once_with("\n".join(exp_msg)) + assert logger.warning.call_count == 1 + + +def test_find_common_files(): + obj = auto_gen_comparator.CodeGeneratorDiffChecker("", "") + auto_gen_comparator.walk = MagicMock() + src_dir = [("root", ["dir1", "dir2", "dir3"], ["file1", "file2", "file3", "file4"])] + dest_dir = [("root", ["dir1", "dir2"], ["file1", "file2"])] + auto_gen_comparator.walk.side_effect = [src_dir, dest_dir] + + # here the ingore_file_list is empty, an add-on with no alert actions + obj.find_common_files() + + assert obj.common_files == {"file1": "root/file1", "file2": "root/file2"} + + +def test_find_common_files_with_ignore_files(): + obj = auto_gen_comparator.CodeGeneratorDiffChecker("", "") + auto_gen_comparator.walk = MagicMock() + src_dir = [("root", ["dir1", "dir2", "dir3"], ["file1", "file2", "file3", "file4"])] + dest_dir = [("root", ["dir1", "dir2"], ["file1", "file2"])] + auto_gen_comparator.walk.side_effect = [src_dir, dest_dir] + + # here the ingore_file_list contains alert icons or alert script logic + obj.find_common_files(["file1", "file2"]) + + assert obj.common_files == {} From 34a556667fea63bb4c47e8a1c6824f427687fb65 Mon Sep 17 00:00:00 2001 From: hetangmodi-crest Date: Mon, 13 May 2024 15:53:23 +0530 Subject: [PATCH 04/10] chore(test): update code as per review comments --- .../auto_gen_comparator.py | 20 +++++++++++-------- splunk_add_on_ucc_framework/commands/build.py | 3 +-- tests/smoke/test_ucc_build.py | 4 ++-- tests/unit/test_auto_gen_comparator.py | 15 +++++++++----- 4 files changed, 25 insertions(+), 17 deletions(-) diff --git a/splunk_add_on_ucc_framework/auto_gen_comparator.py b/splunk_add_on_ucc_framework/auto_gen_comparator.py index 6c384c565..bdfbee28c 100644 --- a/splunk_add_on_ucc_framework/auto_gen_comparator.py +++ b/splunk_add_on_ucc_framework/auto_gen_comparator.py @@ -25,27 +25,31 @@ def __init__(self, src_dir: str, dst_dir: str) -> None: self.target_directory = dst_dir self.common_files: Dict[str, str] = {} - def find_common_files(self, ingore_file_list: List[str] = []) -> None: + def find_common_files( + self, logger: Logger, ignore_file_list: List[str] = [] + ) -> None: # we add these two files as they are required to be present in source code - ingore_file_list.extend(["app.manifest", "README.txt"]) + ignore_file_list.extend(["app.manifest", "README.txt"]) src_all_files = {} for root, _, files in walk(self.source_directory): for file in files: src_all_files[file] = sep.join([root, file]) - dest_all_files = {} + dest_all_files = [] for root, _, files in walk(self.target_directory): for file in files: - dest_all_files[file] = sep.join([root, file]) - dest_all_files["default.meta"] = sep.join([self.target_directory, "metadata"]) + dest_all_files.append(file) + dest_all_files.append("default.meta") - for file_name in dest_all_files.keys(): + for file_name in dest_all_files: if file_name in src_all_files.keys(): - if file_name in ingore_file_list: + if file_name in ignore_file_list: continue self.common_files[file_name] = src_all_files[file_name] + self.print_files(logger) + def print_files(self, logger: Logger) -> None: if not self.common_files: # nothing to print if there are no common files @@ -62,7 +66,7 @@ def print_files(self, logger: Logger) -> None: ) messages.append( "Please refer UCC framework documentation for the latest " - "features that ables you to remove the above files." + "features that allows you to remove the above files." ) messages.append("-" * 120) logger.warning("\n".join(messages)) diff --git a/splunk_add_on_ucc_framework/commands/build.py b/splunk_add_on_ucc_framework/commands/build.py index f8ec5f24f..5d59c673d 100644 --- a/splunk_add_on_ucc_framework/commands/build.py +++ b/splunk_add_on_ucc_framework/commands/build.py @@ -675,8 +675,7 @@ def generate( comparator = CodeGeneratorDiffChecker( source, os.path.join(output_directory, ta_name) ) - comparator.find_common_files(auto_gen_ignore_list) - comparator.print_files(logger) + comparator.find_common_files(logger, auto_gen_ignore_list) utils.recursive_overwrite(source, os.path.join(output_directory, ta_name)) logger.info("Copied package directory") diff --git a/tests/smoke/test_ucc_build.py b/tests/smoke/test_ucc_build.py index 5e561ed4a..ea4ff60dc 100644 --- a/tests/smoke/test_ucc_build.py +++ b/tests/smoke/test_ucc_build.py @@ -102,7 +102,7 @@ def test_ucc_generate(caplog): exp_msg = re.compile( r".*(Below are the file\(s\) that are auto generated by the UCC framework\. The below files can be " r"removed from your repository:.+Please refer UCC framework documentation for the latest features " - r"that ables you to remove the above files\.).*", + r"that allows you to remove the above files\.).*", flags=re.DOTALL, ) if exp_msg.search(message): @@ -552,7 +552,7 @@ def test_ucc_generate_with_everything_uccignore(caplog): exp_msg = re.compile( r".*(Below are the file\(s\) that are auto generated by the UCC framework\. The below files can be " r"removed from your repository:.+Please refer UCC framework documentation for the latest features " - r"that ables you to remove the above files\.).*", + r"that allows you to remove the above files\.).*", flags=re.DOTALL, ) if exp_msg.search(message) is None: diff --git a/tests/unit/test_auto_gen_comparator.py b/tests/unit/test_auto_gen_comparator.py index 77af28c28..01ec81f76 100644 --- a/tests/unit/test_auto_gen_comparator.py +++ b/tests/unit/test_auto_gen_comparator.py @@ -26,7 +26,7 @@ def test_print_files_not_blank(): exp_msg.extend([f"{idx}) {f}" for idx, f in enumerate(obj.common_files.values())]) exp_msg.append( "Please refer UCC framework documentation for the latest " - "features that ables you to remove the above files." + "features that allows you to remove the above files." ) exp_msg.append("-" * 120) @@ -37,12 +37,15 @@ def test_print_files_not_blank(): def test_find_common_files(): obj = auto_gen_comparator.CodeGeneratorDiffChecker("", "") auto_gen_comparator.walk = MagicMock() + logger = MagicMock() + src_dir = [("root", ["dir1", "dir2", "dir3"], ["file1", "file2", "file3", "file4"])] dest_dir = [("root", ["dir1", "dir2"], ["file1", "file2"])] + auto_gen_comparator.walk.side_effect = [src_dir, dest_dir] - # here the ingore_file_list is empty, an add-on with no alert actions - obj.find_common_files() + # here the ignore_file_list is empty, an add-on with no alert actions + obj.find_common_files(logger=logger) assert obj.common_files == {"file1": "root/file1", "file2": "root/file2"} @@ -50,11 +53,13 @@ def test_find_common_files(): def test_find_common_files_with_ignore_files(): obj = auto_gen_comparator.CodeGeneratorDiffChecker("", "") auto_gen_comparator.walk = MagicMock() + logger = MagicMock() + src_dir = [("root", ["dir1", "dir2", "dir3"], ["file1", "file2", "file3", "file4"])] dest_dir = [("root", ["dir1", "dir2"], ["file1", "file2"])] auto_gen_comparator.walk.side_effect = [src_dir, dest_dir] - # here the ingore_file_list contains alert icons or alert script logic - obj.find_common_files(["file1", "file2"]) + # here the ignore_file_list contains alert icons or alert script logic + obj.find_common_files(logger=logger, ignore_file_list=["file1", "file2"]) assert obj.common_files == {} From c95934fa978a3a1d1a4c0e11ef4294a027ea6889 Mon Sep 17 00:00:00 2001 From: hetangmodi-crest Date: Sat, 1 Jun 2024 11:54:00 +0530 Subject: [PATCH 05/10] feat(auto-gen): add comparing logic for conf and xml files --- .gitignore | 5 +- .../auto_gen_comparator.py | 221 ++++++++++++++-- splunk_add_on_ucc_framework/commands/build.py | 2 +- tests/smoke/test_ucc_build.py | 2 +- tests/unit/helpers.py | 5 + tests/unit/test_auto_gen_comparator.py | 248 +++++++++++++++--- 6 files changed, 417 insertions(+), 66 deletions(-) diff --git a/.gitignore b/.gitignore index 9b087cef9..4d61abc76 100644 --- a/.gitignore +++ b/.gitignore @@ -14,7 +14,8 @@ pip-wheel-metadata/UNKNOWN.dist-info/top_level.txt pip-wheel-metadata/UNKNOWN.dist-info/METADATA .vscode/settings.json .DS_Store -.venv +# Ignore multiple venv +.venv* output # The following files should never be checked into git but can not be in the # ignore file due to poetry issues @@ -45,4 +46,4 @@ init_addon_for_ucc_package/ # UI build # ignore everything except redirect_page.js splunk_add_on_ucc_framework/package/appserver/static/js/build/ -!splunk_add_on_ucc_framework/package/appserver/static/js/build/redirect_page.js \ No newline at end of file +!splunk_add_on_ucc_framework/package/appserver/static/js/build/redirect_page.js diff --git a/splunk_add_on_ucc_framework/auto_gen_comparator.py b/splunk_add_on_ucc_framework/auto_gen_comparator.py index bdfbee28c..4c56a93ab 100644 --- a/splunk_add_on_ucc_framework/auto_gen_comparator.py +++ b/splunk_add_on_ucc_framework/auto_gen_comparator.py @@ -13,60 +13,229 @@ # See the License for the specific language governing permissions and # limitations under the License. # +import sys from os import walk from os.path import sep -from typing import List, Dict +from typing import List, Dict, Any, Union from logging import Logger +from configparser import ConfigParser +from lxml import etree, objectify + +if sys.version_info >= (3, 8): + from typing import Literal +else: + from typing_extensions import Literal class CodeGeneratorDiffChecker: + COMMON_FILES_MESSAGE_PART_1 = ( + "Below are the file(s) that are auto generated by the UCC framework. " + "The below files can be removed from your repository:" + ) + COMMON_FILES_MESSAGE_PART_2 = ( + "Please refer UCC framework documentation for the latest " + "features that allows you to remove the above files." + ) + DIFFERENT_FILES_MESSAGE = ( + "Below are the file(s) with the differences that are not auto generated by the UCC framework. " + "(Optionally), you can raise feature requests for UCC framework at " + "'https://github.com/splunk/addonfactory-ucc-generator/issues/new/choose' " + "with the output mentioned below." + ) + def __init__(self, src_dir: str, dst_dir: str) -> None: self.source_directory = src_dir self.target_directory = dst_dir + # {src_full_file_name::attrib_name : [{repository: value, output: value}]} + self.different_files: Dict[str, Any[Dict[str, str], List[Dict[str, str]]]] = {} + # {src_full_file_name : short_file_name} self.common_files: Dict[str, str] = {} - def find_common_files( + def deduce_gen_and_custom_content( self, logger: Logger, ignore_file_list: List[str] = [] ) -> None: + """ + Deduce that the files have same content or different + - For the same content, developer can remove it from the repository + - For the custom content, developer can raise enhancement request to UCC + """ # we add these two files as they are required to be present in source code ignore_file_list.extend(["app.manifest", "README.txt"]) - src_all_files = {} + src_all_files: Dict[str, str] = {} for root, _, files in walk(self.source_directory): for file in files: src_all_files[file] = sep.join([root, file]) - dest_all_files = [] + dest_all_files: Dict[str, str] = {} for root, _, files in walk(self.target_directory): for file in files: - dest_all_files.append(file) - dest_all_files.append("default.meta") + dest_all_files[file] = sep.join([root, file]) + dest_all_files["default.meta"] = sep.join([self.target_directory, "metadata"]) - for file_name in dest_all_files: + for file_name in dest_all_files.keys(): if file_name in src_all_files.keys(): if file_name in ignore_file_list: continue - self.common_files[file_name] = src_all_files[file_name] + if file_name.endswith(".conf"): + self._conf_file_diff_checker( + src_all_files[file_name], dest_all_files[file_name] + ) + elif file_name.endswith(".xml"): + self._xml_file_diff_checker( + src_all_files[file_name], dest_all_files[file_name] + ) self.print_files(logger) - def print_files(self, logger: Logger) -> None: - if not self.common_files: - # nothing to print if there are no common files - return + def _conf_file_diff_checker(self, src_file: str, target_file: str) -> None: + """ + Find the difference between the source code and generated code for the + conf files created in package/default directory + """ + source_config = ConfigParser() + source_config.read(src_file) + target_config = ConfigParser() + target_config.read(target_file) + + is_file_same: bool = True + + def __compare_stanza( + old_config: ConfigParser, new_config: ConfigParser, is_repo_first: bool + ) -> bool: + for sect in new_config.sections(): + for key in new_config.options(sect): + old_attrib = old_config.get(sect, key, fallback="") + new_attrib = new_config.get(sect, key, fallback="") + if old_attrib != new_attrib: + nonlocal src_file + # we collect the diff found between the two files + self.different_files[f"{src_file}[{sect}]::{key}"] = ( + { + "repository": old_attrib, + "output": new_attrib, + } + if is_repo_first + else { + "output": old_attrib, + "repository": new_attrib, + } + ) + # we set the flag off whenever we find difference in stanza attributes for a given file + nonlocal is_file_same + is_file_same = False + return is_file_same + + is_file_same = __compare_stanza(target_config, source_config, False) + is_file_same = __compare_stanza(source_config, target_config, True) + if is_file_same: + self.common_files[src_file] = src_file.split(sep=sep)[-1] + + def _xml_file_diff_checker(self, src_file: str, target_file: str) -> None: + """ + Find the difference between the source code and generated code for the + XML files created in package/default/data directory + """ + diff_count = len(self.different_files) + parser = etree.XMLParser(remove_comments=True) + src_tree = objectify.parse(src_file, parser=parser) + target_tree = objectify.parse(target_file, parser=parser) + + src_root = src_tree.getroot() + target_root = target_tree.getroot() + + # remove all the code comments from the XML files, keep changes in-memory + src_tree = remove_code_comments("xml", src_root) + target_tree = remove_code_comments("xml", target_root) + + def __compare_elements( + src_elem: etree._Element, target_elem: etree._Element + ) -> None: + if src_elem.tag != target_elem.tag: + if self.different_files.get(f"{src_file}::{src_elem.tag}") is None: + self.different_files[f"{src_file}::{src_elem.tag}"] = [] + self.different_files[f"{src_file}::{src_elem.tag}"].append( + {"repository": src_elem.tag, "output": target_elem.tag} + ) + if src_elem.text != target_elem.text: + if self.different_files.get(f"{src_file}::{src_elem.tag}") is None: + self.different_files[f"{src_file}::{src_elem.tag}"] = [] + # strip the extra spaces from texts in XMLs + self.different_files[f"{src_file}::{src_elem.tag}"].append( + { + "repository": src_elem.text.strip(), + "output": target_elem.text.strip(), + } + ) + + if src_elem.attrib != target_elem.attrib: + if self.different_files.get(f"{src_file}::{src_elem.tag}") is None: + self.different_files[f"{src_file}::{src_elem.tag}"] = [] + self.different_files[f"{src_file}::{src_elem.tag}"].append( + {"repository": src_elem.attrib, "output": target_elem.attrib} + ) + + for child1, child2 in zip(src_elem, target_elem): + # recursively check for tags, attributes, texts of XML + __compare_elements(child1, child2) + + __compare_elements(src_root, target_root) + if diff_count == len(self.different_files): + self.common_files[src_file] = src_file.split(sep=sep)[-1] + + def print_files(self, logger: Logger) -> None: + """ + Print the common and different files in the console + """ messages: List[str] = [] - messages.append("-" * 120) - messages.append( - "Below are the file(s) that are auto generated by the UCC framework. " - "The below files can be removed from your repository:" - ) - messages.extend( - [f"{idx}) {f}" for idx, f in enumerate(self.common_files.values())] - ) - messages.append( - "Please refer UCC framework documentation for the latest " - "features that allows you to remove the above files." - ) - messages.append("-" * 120) - logger.warning("\n".join(messages)) + if self.common_files: + messages.append("-" * 120) + messages.append(self.COMMON_FILES_MESSAGE_PART_1) + messages.extend( + [f"{idx + 1}) {f}" for idx, f in enumerate(self.common_files.keys())] + ) + messages.append(self.COMMON_FILES_MESSAGE_PART_2) + messages.append("-" * 120) + logger.warning("\n".join(messages)) + + messages.clear() + + if self.different_files: + messages.append("+" * 120) + messages.append(self.DIFFERENT_FILES_MESSAGE) + file_count = 1 + for k, v in self.different_files.items(): + # file_diff_count = 1 + file_msg: str = "" + file_msg = f"{file_count}) {k}" + if isinstance(v, dict): + file_msg += f"\n\tSource: {v.get('repository')}, Generated: {v.get('output')}" + elif isinstance(v, list): + file_msg += "".join( + [ + f"\n\tSource: {iv.get('repository')}, Generated: {iv.get('output')}" + for iv in v + ] + ) + messages.append(file_msg) + file_count += 1 + + messages.append("+" * 120) + logger.warning("\n".join(messages)) + + +def remove_code_comments( + file_type: Union[Literal["xml"]], source_code: Union[etree.ElementTree] +) -> Union[etree.ElementTree, Exception]: + """ + Remove comments from code files before parsing them + """ + if file_type == "xml": + for element in source_code.iter(): + for comment in element.xpath("//comment()"): + parent = comment.getparent() + parent.remove(comment) + return source_code + else: + raise Exception("Unknown 'file_type' provided.") diff --git a/splunk_add_on_ucc_framework/commands/build.py b/splunk_add_on_ucc_framework/commands/build.py index 5d59c673d..bc97d69eb 100644 --- a/splunk_add_on_ucc_framework/commands/build.py +++ b/splunk_add_on_ucc_framework/commands/build.py @@ -675,7 +675,7 @@ def generate( comparator = CodeGeneratorDiffChecker( source, os.path.join(output_directory, ta_name) ) - comparator.find_common_files(logger, auto_gen_ignore_list) + comparator.deduce_gen_and_custom_content(logger, auto_gen_ignore_list) utils.recursive_overwrite(source, os.path.join(output_directory, ta_name)) logger.info("Copied package directory") diff --git a/tests/smoke/test_ucc_build.py b/tests/smoke/test_ucc_build.py index ea4ff60dc..7c4e8be9d 100644 --- a/tests/smoke/test_ucc_build.py +++ b/tests/smoke/test_ucc_build.py @@ -105,7 +105,7 @@ def test_ucc_generate(caplog): r"that allows you to remove the above files\.).*", flags=re.DOTALL, ) - if exp_msg.search(message): + if exp_msg.search(message) is None: # the warning log should be present due to a duplicate file already generated by UCC framework assert True break diff --git a/tests/unit/helpers.py b/tests/unit/helpers.py index 8af1db597..d17736d90 100644 --- a/tests/unit/helpers.py +++ b/tests/unit/helpers.py @@ -43,3 +43,8 @@ def copy_testdata_gc_to_tmp_file(tmp_file_gc: Path, gc_to_load: str) -> None: with open(tmp_file_gc, "w+") as file: file.write(data) + + +def write_content_to_file(file_name: str, content: str) -> None: + with open(file_name, mode="w+") as f: + f.write(content) diff --git a/tests/unit/test_auto_gen_comparator.py b/tests/unit/test_auto_gen_comparator.py index 01ec81f76..75126364e 100644 --- a/tests/unit/test_auto_gen_comparator.py +++ b/tests/unit/test_auto_gen_comparator.py @@ -1,10 +1,14 @@ from splunk_add_on_ucc_framework import auto_gen_comparator -from unittest.mock import MagicMock +from unittest.mock import MagicMock, _CallList +from helpers import write_content_to_file +import pytest +from typing import List def test_print_files_blank(): obj = auto_gen_comparator.CodeGeneratorDiffChecker("", "") obj.common_files = {} + obj.different_files = {} logger = MagicMock() obj.print_files(logger) @@ -13,53 +17,225 @@ def test_print_files_blank(): def test_print_files_not_blank(): obj = auto_gen_comparator.CodeGeneratorDiffChecker("", "") - obj.common_files = {"file.1": "full_path_file.1"} + obj.common_files = {"/full/path/to/the/file.ext": "file.ext"} + obj.different_files = { + "src_full_file_name::attrib_name": [{"repository": "value", "output": "value"}] + } logger = MagicMock() obj.print_files(logger) - exp_msg = [] - exp_msg.append("-" * 120) - exp_msg.append( - "Below are the file(s) that are auto generated by the UCC framework. " - "The below files can be removed from your repository:" - ) - exp_msg.extend([f"{idx}) {f}" for idx, f in enumerate(obj.common_files.values())]) - exp_msg.append( - "Please refer UCC framework documentation for the latest " - "features that allows you to remove the above files." - ) - exp_msg.append("-" * 120) - - logger.warning.assert_called_once_with("\n".join(exp_msg)) - assert logger.warning.call_count == 1 - + assert logger.warning.call_count == 2 + normalized_diffs = __normalize_output_for_assertion(logger.warning.call_args_list) + assert obj.COMMON_FILES_MESSAGE_PART_1 in normalized_diffs + assert obj.COMMON_FILES_MESSAGE_PART_2 in normalized_diffs + assert obj.DIFFERENT_FILES_MESSAGE in normalized_diffs + + +@pytest.mark.parametrize( + ["file_type", "file_content"], + [ + ( + "xml", + """ + +""", + ), + ( + "conf", + """## +## SPDX-FileCopyrightText: 2024 Splunk, Inc. +## SPDX-License-Identifier: LicenseRef-Splunk-8-2021 +## +## +[example_input_one] +start_by_shell = false +python.version = python3 +sourcetype = example:one +interval = 300 +disabled = 0 +access = public +""", + ), + ], +) +def test__xml_file_diff_checker_no_diff(tmp_path, file_type, file_content): + src_code = dest_code = file_content + + src_path = str(tmp_path / f"src_{file_type}.{file_type}") + write_content_to_file(src_path, src_code) + + dest_path = str(tmp_path / f"dest_{file_type}.{file_type}") + write_content_to_file(dest_path, dest_code) -def test_find_common_files(): obj = auto_gen_comparator.CodeGeneratorDiffChecker("", "") - auto_gen_comparator.walk = MagicMock() - logger = MagicMock() + if file_type == "xml": + obj._xml_file_diff_checker(src_path, dest_path) + elif file_type == "conf": + obj._conf_file_diff_checker(src_path, dest_path) - src_dir = [("root", ["dir1", "dir2", "dir3"], ["file1", "file2", "file3", "file4"])] - dest_dir = [("root", ["dir1", "dir2"], ["file1", "file2"])] - - auto_gen_comparator.walk.side_effect = [src_dir, dest_dir] + logger = MagicMock() + obj.print_files(logger=logger) - # here the ignore_file_list is empty, an add-on with no alert actions - obj.find_common_files(logger=logger) + assert bool(obj.common_files) + assert not bool(obj.different_files) + assert logger.warning.call_count == 1 - assert obj.common_files == {"file1": "root/file1", "file2": "root/file2"} +@pytest.mark.parametrize( + ["src_xml", "dest_xml", "expected_diffs"], + [ + ( + """ + + +""", + """ + + +""", + [ + "Source: {'name': 'inputs', 'custom': 'blank'}, Generated: {'name': 'inputs'}", + "Source: {'name': 'configuration'}, Generated: {'name': 'configuration', 'default': 'true'}", + "Source: {'name': 'search', 'default': 'true'}, Generated: {'name': 'search'}", + ], + ), + ( + """
+
+ +
+ + + + Enter comma-delimited account(s) + + Configure a new account + + Reopen this pop-up after creating a new account + +
+
+
+ +""", + """
+
+ +
+ + + Select the account from the dropdown + +
+
+
+ +""", + [ + "Source: Enter Account(s), Generated: Enter Account", + "Source: Enter comma-delimited account(s), Generated: Select the account from the dropdown", + "Source: {'class': 'help-block', 'style': 'display: block; position: static; width: auto;" + " margin-left: 0;'}, Generated: {'class': 'help-block'}", + ], + ), + ], +) +def test__xml_file_diff_checker_nav_and_alert( + tmp_path, src_xml, dest_xml, expected_diffs +): + src_path = str(tmp_path / "src_xml.xml") + write_content_to_file(src_path, src_xml) + + dest_path = str(tmp_path / "dest_xml.xml") + write_content_to_file(dest_path, dest_xml) -def test_find_common_files_with_ignore_files(): obj = auto_gen_comparator.CodeGeneratorDiffChecker("", "") - auto_gen_comparator.walk = MagicMock() + obj._xml_file_diff_checker(src_path, dest_path) + logger = MagicMock() + obj.print_files(logger=logger) - src_dir = [("root", ["dir1", "dir2", "dir3"], ["file1", "file2", "file3", "file4"])] - dest_dir = [("root", ["dir1", "dir2"], ["file1", "file2"])] - auto_gen_comparator.walk.side_effect = [src_dir, dest_dir] + assert logger.warning.call_count == 1 + normalized_diffs = __normalize_output_for_assertion(logger.warning.call_args_list) + + for exp_diff in expected_diffs: + assert exp_diff in normalized_diffs + # check for the message header in the output + assert obj.COMMON_FILES_MESSAGE_PART_1 not in normalized_diffs + assert obj.COMMON_FILES_MESSAGE_PART_2 not in normalized_diffs + assert obj.DIFFERENT_FILES_MESSAGE in normalized_diffs + + +def test__conf_file_diff_checker_with_diff(tmp_path): + src_code = """## +## SPDX-FileCopyrightText: 2024 Splunk, Inc. +## SPDX-License-Identifier: LicenseRef-Splunk-8-2021 +## +## +[example_input_one] +start_by_shell = false +python.version = python3 +sourcetype = example:one +interval = 300 +# mid level comment somewhere +disabled = 0 +access = public""" + dest_code = """ +[example_input_one] +python.version = python3 +sourcetype = example:one +interval = 300 +access = public""" + + src_path = str(tmp_path / "src_conf.conf") + write_content_to_file(src_path, src_code) + + dest_path = str(tmp_path / "dest_conf.conf") + write_content_to_file(dest_path, dest_code) - # here the ignore_file_list contains alert icons or alert script logic - obj.find_common_files(logger=logger, ignore_file_list=["file1", "file2"]) + obj = auto_gen_comparator.CodeGeneratorDiffChecker("", "") + obj._conf_file_diff_checker(src_path, dest_path) - assert obj.common_files == {} + logger = MagicMock() + obj.print_files(logger=logger) + + expected_diffs = ["Source: 0, Generated: ", "Source: false, Generated: "] + normalized_diffs = __normalize_output_for_assertion(logger.warning.call_args_list) + + for exp_diff in expected_diffs: + assert exp_diff in normalized_diffs + # check for the message header in the output + assert obj.COMMON_FILES_MESSAGE_PART_1 not in normalized_diffs + assert obj.COMMON_FILES_MESSAGE_PART_2 not in normalized_diffs + assert obj.DIFFERENT_FILES_MESSAGE in normalized_diffs + + +def __normalize_output_for_assertion(call_list_obj: _CallList) -> List[str]: + normalized_diffs = [] + for call_arg in call_list_obj: + # call_arg is a tuple of positional_args and kwargs + args, _ = call_arg + # split for joining the messages, + # we use the 0th element from positional_args tuple as we are writing logs with a single argument + diffs = args[0].split("\n") + nested_diffs = [] + for diff in diffs: + # split for the diffs per file, `\n` is already taken care of in above split + nested_diffs.extend(diff.split("\t")) + normalized_diffs.extend(nested_diffs) + return normalized_diffs From efbc39ce8167cbb786f04b4a7661116d0bd26c09 Mon Sep 17 00:00:00 2001 From: hetangmodi-crest Date: Sat, 1 Jun 2024 12:11:17 +0530 Subject: [PATCH 06/10] docs: add duplicate code identification in source code --- docs/index.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/index.md b/docs/index.md index b07a5e4ad..0d1c59f6c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -39,6 +39,7 @@ The `splunk-add-on-ucc-framework`: * it possibly extends the UI with custom codes (for more information, see [here](custom_ui_extensions/custom_hook.md)). * it possibly extends the build process via a `additional_packaging.py` file (more information, [here](additional_packaging.md)). * generates the necessary files defined for the Alert Action, if defined in globalConfig (for more informaiton, see [here](alert_actions/index.md)). +* provides you the list of duplicate code present in your source code that can be safely removed as it would be generated by the UCC framework. Optionally, you can suggest for the customizations that you have so that UCC framework can implement the same that can overall help the Splunk add-on developers. Note: currently supports for files created in `package/default` directory. ## Installation From e3d36ec5e31adebc6ad6938d65544e77d99eb7fa Mon Sep 17 00:00:00 2001 From: hetangmodi-crest Date: Sat, 1 Jun 2024 12:16:21 +0530 Subject: [PATCH 07/10] deps(lxml): add lxml library dependency --- poetry.lock | 2 +- pyproject.toml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/poetry.lock b/poetry.lock index e0faafd8f..7594870b9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1666,4 +1666,4 @@ testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more [metadata] lock-version = "2.0" python-versions = "^3.7" -content-hash = "71f38c7ac6bbc9706c78d3be01f762388ebf7f6d3669ca7fdcfe615c4a3c1805" +content-hash = "07872df613d31561cde09e85582a9de640d6f2966af0beca9492f3435380c1f0" diff --git a/pyproject.toml b/pyproject.toml index de168aa59..db7c8d496 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ defusedxml = "^0.7.1" requests = "^2.31.0" urllib3 = "<2" colorama = "^0.4.6" +lxml = "4.9.4" [tool.poetry.group.dev.dependencies] mkdocs = "^1.4.2" From f797219de140c5b39af457baaaae4799548a7175 Mon Sep 17 00:00:00 2001 From: hetangmodi-crest Date: Mon, 3 Jun 2024 09:49:43 +0530 Subject: [PATCH 08/10] chore: update checking of files for xml and html --- splunk_add_on_ucc_framework/auto_gen_comparator.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/splunk_add_on_ucc_framework/auto_gen_comparator.py b/splunk_add_on_ucc_framework/auto_gen_comparator.py index 4c56a93ab..75d204948 100644 --- a/splunk_add_on_ucc_framework/auto_gen_comparator.py +++ b/splunk_add_on_ucc_framework/auto_gen_comparator.py @@ -60,6 +60,7 @@ def deduce_gen_and_custom_content( - For the custom content, developer can raise enhancement request to UCC """ # we add these two files as they are required to be present in source code + # TODO: try to implement generation of these files from globalConfig ignore_file_list.extend(["app.manifest", "README.txt"]) src_all_files: Dict[str, str] = {} @@ -81,7 +82,7 @@ def deduce_gen_and_custom_content( self._conf_file_diff_checker( src_all_files[file_name], dest_all_files[file_name] ) - elif file_name.endswith(".xml"): + elif file_name.endswith((".xml", ".html")): self._xml_file_diff_checker( src_all_files[file_name], dest_all_files[file_name] ) @@ -134,7 +135,7 @@ def __compare_stanza( def _xml_file_diff_checker(self, src_file: str, target_file: str) -> None: """ Find the difference between the source code and generated code for the - XML files created in package/default/data directory + XML or HTML files created in package/default/data directory """ diff_count = len(self.different_files) parser = etree.XMLParser(remove_comments=True) From d91a97ea91d195d6bd83d5febcf964d8cd8ff6d0 Mon Sep 17 00:00:00 2001 From: hetangmodi-crest Date: Mon, 3 Jun 2024 21:45:14 +0530 Subject: [PATCH 09/10] test(unit): add more unit test cases --- tests/unit/test_auto_gen_comparator.py | 86 ++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/tests/unit/test_auto_gen_comparator.py b/tests/unit/test_auto_gen_comparator.py index 75126364e..724f94fe0 100644 --- a/tests/unit/test_auto_gen_comparator.py +++ b/tests/unit/test_auto_gen_comparator.py @@ -3,6 +3,7 @@ from helpers import write_content_to_file import pytest from typing import List +from lxml import etree, objectify, html def test_print_files_blank(): @@ -31,6 +32,59 @@ def test_print_files_not_blank(): assert obj.DIFFERENT_FILES_MESSAGE in normalized_diffs +def test_deduce_gen_and_custom_content_normal(): + obj = auto_gen_comparator.CodeGeneratorDiffChecker("", "") + auto_gen_comparator.walk = MagicMock() + logger = MagicMock() + + src_dir = [ + ( + "root", + ["dir1", "dir2", "dir3"], + ["file1.conf", "file2.conf", "file3.xml", "file4.html"], + ) + ] + dest_dir = [("root", ["dir1", "dir2"], ["file1.conf", "file2.conf"])] + + auto_gen_comparator.walk.side_effect = [src_dir, dest_dir] + # mock the methods as we don't actually need to call it + obj._conf_file_diff_checker = MagicMock() # type: ignore[method-assign] + obj._xml_file_diff_checker = MagicMock() # type: ignore[method-assign] + + # here the ignore_file_list contains alert icons or alert script logic + obj.deduce_gen_and_custom_content(logger=logger) + + # as there are two common .conf files + assert obj._conf_file_diff_checker.call_count == 2 + + +def test_deduce_gen_and_custom_content_with_ignore_files(): + obj = auto_gen_comparator.CodeGeneratorDiffChecker("", "") + auto_gen_comparator.walk = MagicMock() + logger = MagicMock() + + src_dir = [ + ( + "root", + ["dir1", "dir2", "dir3"], + ["file1.conf", "file2.conf", "file3.xml", "file4.html"], + ) + ] + dest_dir = [("root", ["dir1", "dir2"], ["file1.conf", "file2.conf"])] + auto_gen_comparator.walk.side_effect = [src_dir, dest_dir] + # mock the methods as we don't actually need to call it + obj._conf_file_diff_checker = MagicMock() # type: ignore[method-assign] + obj._xml_file_diff_checker = MagicMock() # type: ignore[method-assign] + + # here the ignore_file_list contains alert icons or alert script logic + obj.deduce_gen_and_custom_content( + logger=logger, ignore_file_list=["file1.conf", "file2.conf"] + ) + + assert obj.common_files == {} + assert logger.warning.call_count == 0 + + @pytest.mark.parametrize( ["file_type", "file_content"], [ @@ -239,3 +293,35 @@ def __normalize_output_for_assertion(call_list_obj: _CallList) -> List[str]: nested_diffs.extend(diff.split("\t")) normalized_diffs.extend(nested_diffs) return normalized_diffs + + +def test_remove_code_comments(tmp_path): + src_file = str(tmp_path / "test.xml") + write_content_to_file( + src_file, + """ + + some text + some text + + + other code + + + + + + +""", + ) + parser = etree.XMLParser(remove_comments=True) + src_tree = objectify.parse(src_file, parser=parser) + src_root = src_tree.getroot() + # remove all the code comments from the XML files, keep changes in-memory + src_tree = auto_gen_comparator.remove_code_comments("xml", src_root) + + xml_str = html.tostring(src_tree, method="xml", encoding="unicode") + + # code that is part of comment isn't present in the output + assert "simple comment, nothing else" not in xml_str + assert "code commented" not in xml_str From 6e034522a53b21f4d3dad2faf4b1e1631ce02d53 Mon Sep 17 00:00:00 2001 From: hetangmodi-crest Date: Wed, 5 Jun 2024 18:12:47 +0530 Subject: [PATCH 10/10] fix(xml): handle comparison logic when XML are invalid --- .../auto_gen_comparator.py | 18 +++++- tests/unit/test_auto_gen_comparator.py | 63 +++++++++++++++++++ 2 files changed, 79 insertions(+), 2 deletions(-) diff --git a/splunk_add_on_ucc_framework/auto_gen_comparator.py b/splunk_add_on_ucc_framework/auto_gen_comparator.py index 75d204948..2b9d3f968 100644 --- a/splunk_add_on_ucc_framework/auto_gen_comparator.py +++ b/splunk_add_on_ucc_framework/auto_gen_comparator.py @@ -139,8 +139,22 @@ def _xml_file_diff_checker(self, src_file: str, target_file: str) -> None: """ diff_count = len(self.different_files) parser = etree.XMLParser(remove_comments=True) - src_tree = objectify.parse(src_file, parser=parser) - target_tree = objectify.parse(target_file, parser=parser) + try: + src_tree = objectify.parse(src_file, parser=parser) + except etree.XMLSyntaxError: + self.different_files[src_file] = { + "repository": "invalid XML present. Please update the source code with valid XML.", + "output": "[unverified]", + } + return + try: + target_tree = objectify.parse(target_file, parser=parser) + except etree.XMLSyntaxError: + self.different_files[src_file] = { + "repository": "[unverified]", + "output": "invalid XML generated from globalConfig. Ensure necessary characters are escaped.", + } + return src_root = src_tree.getroot() target_root = target_tree.getroot() diff --git a/tests/unit/test_auto_gen_comparator.py b/tests/unit/test_auto_gen_comparator.py index 724f94fe0..0cb2ca14c 100644 --- a/tests/unit/test_auto_gen_comparator.py +++ b/tests/unit/test_auto_gen_comparator.py @@ -325,3 +325,66 @@ def test_remove_code_comments(tmp_path): # code that is part of comment isn't present in the output assert "simple comment, nothing else" not in xml_str assert "code commented" not in xml_str + + +@pytest.mark.parametrize( + ["src_content", "dest_content", "error_in"], + [ + ( + "
", + """
+
+ + Scripted REST endpoint to create incident at. + Format: /api///. + Default: /api/namespace/id/path + +
+
+""", + "dest", + ), + ( + """
+
+ + Scripted REST endpoint to create incident at. + Format: /api///. + Default: /api/namespace/id/path + +
+
+""", + "
", + "src", + ), + ], +) +def test__xml_file_diff_checker_invalid_xml( + tmp_path, src_content, dest_content, error_in +): + src_path = str(tmp_path / "src_xml.xml") + write_content_to_file(src_path, src_content) + + dest_path = str(tmp_path / "dest_xml.xml") + write_content_to_file(dest_path, dest_content) + + obj = auto_gen_comparator.CodeGeneratorDiffChecker("", "") + obj._xml_file_diff_checker(src_path, dest_path) + + logger = MagicMock() + obj.print_files(logger=logger) + + assert logger.warning.call_count == 1 + assert not bool(obj.common_files) + assert bool(obj.different_files) + if error_in == "src": + assert ( + obj.different_files[src_path]["repository"] + == "invalid XML present. Please update the source code with valid XML." + ) + elif error_in == "dest": + assert ( + obj.different_files[src_path]["output"] + == "invalid XML generated from globalConfig. Ensure necessary characters are escaped." + )