From a35b5e0c552e6b58b9a465fcf09db50b066c5f1a Mon Sep 17 00:00:00 2001 From: pyth0n1c Date: Fri, 17 Jan 2025 15:24:28 -0800 Subject: [PATCH] write all objects to conf file at once instead of one at a time. this reverts the behavior to what it used to be like. also, fix the filename written out in transforms.j2 for a file-backed lookup such that it contains the datetime stamp of the modifications. finally, ruff reformatted the conf_writer.py file, so lots of formatting changes. --- contentctl/output/conf_writer.py | 335 +++++++++++++--------- contentctl/output/templates/transforms.j2 | 4 +- 2 files changed, 203 insertions(+), 136 deletions(-) diff --git a/contentctl/output/conf_writer.py b/contentctl/output/conf_writer.py index 31b776b2..bcfb6d19 100644 --- a/contentctl/output/conf_writer.py +++ b/contentctl/output/conf_writer.py @@ -1,16 +1,17 @@ -from typing import Any, Sequence +import configparser import datetime -import re -import os import json -import configparser -from xmlrpc.client import APPLICATION_ERROR -from jinja2 import Environment, FileSystemLoader, StrictUndefined +import os import pathlib -from contentctl.objects.security_content_object import SecurityContentObject -from contentctl.objects.dashboard import Dashboard -from contentctl.objects.config import build, CustomApp +import re import xml.etree.ElementTree as ET +from typing import Any, Sequence + +from jinja2 import Environment, FileSystemLoader, StrictUndefined + +from contentctl.objects.config import CustomApp, build +from contentctl.objects.dashboard import Dashboard +from contentctl.objects.security_content_object import SecurityContentObject # This list is not exhaustive of all default conf files, but should be # sufficient for our purposes. @@ -82,59 +83,68 @@ "workload_rules.conf", ] -class ConfWriter(): +class ConfWriter: @staticmethod - def custom_jinja2_enrichment_filter(string:str, object:SecurityContentObject): + def custom_jinja2_enrichment_filter(string: str, object: SecurityContentObject): substitutions = re.findall(r"%[^%]*%", string) updated_string = string for sub in substitutions: - sub_without_percents = sub.replace("%","") + sub_without_percents = sub.replace("%", "") if hasattr(object, sub_without_percents): - updated_string = updated_string.replace(sub, str(getattr(object, sub_without_percents))) - elif hasattr(object,'tags') and hasattr(object.tags, sub_without_percents): - updated_string = updated_string.replace(sub, str(getattr(object.tags, sub_without_percents))) + updated_string = updated_string.replace( + sub, str(getattr(object, sub_without_percents)) + ) + elif hasattr(object, "tags") and hasattr(object.tags, sub_without_percents): + updated_string = updated_string.replace( + sub, str(getattr(object.tags, sub_without_percents)) + ) else: raise Exception(f"Unable to find field {sub} in object {object.name}") - + return updated_string - + @staticmethod - def escapeNewlines(obj:Any): + def escapeNewlines(obj: Any): # Ensure that any newlines that occur in a string are escaped with a \. # Failing to do so will result in an improperly formatted conf files that # cannot be parsed - if isinstance(obj,str): - # Remove leading and trailing characters. Conf parsers may erroneously - # Parse fields if they have leading or trailing newlines/whitespace and we + if isinstance(obj, str): + # Remove leading and trailing characters. Conf parsers may erroneously + # Parse fields if they have leading or trailing newlines/whitespace and we # probably don't want that anyway as it doesn't look good in output - return obj.strip().replace(f"\n"," \\\n") + return obj.strip().replace("\n", " \\\n") else: return obj - @staticmethod - def writeConfFileHeader(app_output_path:pathlib.Path, config: build) -> pathlib.Path: - output = ConfWriter.writeFileHeader(app_output_path, config) - - output_path = config.getPackageDirectoryPath()/app_output_path + def writeConfFileHeader( + app_output_path: pathlib.Path, config: build + ) -> pathlib.Path: + output = ConfWriter.writeFileHeader(app_output_path, config) + + output_path = config.getPackageDirectoryPath() / app_output_path output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, 'w') as f: - output = output.encode('utf-8', 'ignore').decode('utf-8') + with open(output_path, "w") as f: + output = output.encode("utf-8", "ignore").decode("utf-8") f.write(output) - #Ensure that the conf file we just generated/update is syntactically valid - ConfWriter.validateConfFile(output_path) + # Ensure that the conf file we just generated/update is syntactically valid + ConfWriter.validateConfFile(output_path) return output_path @staticmethod - def getCustomConfFileStems(config:build)->list[str]: + def getCustomConfFileStems(config: build) -> list[str]: # Get all the conf files in the default directory. We must make a reload.conf_file = simple key/value for them if # they are custom conf files - default_path = config.getPackageDirectoryPath()/"default" + default_path = config.getPackageDirectoryPath() / "default" conf_files = default_path.glob("*.conf") - - custom_conf_file_stems = [conf_file.stem for conf_file in conf_files if conf_file.name not in DEFAULT_CONF_FILES] + + custom_conf_file_stems = [ + conf_file.stem + for conf_file in conf_files + if conf_file.name not in DEFAULT_CONF_FILES + ] return sorted(custom_conf_file_stems) @staticmethod @@ -145,16 +155,17 @@ def writeServerConf(config: build) -> pathlib.Path: j2_env = ConfWriter.getJ2Environment() template = j2_env.get_template(template_name) - output = template.render(custom_conf_files=ConfWriter.getCustomConfFileStems(config)) - - output_path = config.getPackageDirectoryPath()/app_output_path + output = template.render( + custom_conf_files=ConfWriter.getCustomConfFileStems(config) + ) + + output_path = config.getPackageDirectoryPath() / app_output_path output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, 'a') as f: - output = output.encode('utf-8', 'ignore').decode('utf-8') + with open(output_path, "a") as f: + output = output.encode("utf-8", "ignore").decode("utf-8") f.write(output) return output_path - @staticmethod def writeAppConf(config: build) -> pathlib.Path: app_output_path = pathlib.Path("default/app.conf") @@ -163,144 +174,195 @@ def writeAppConf(config: build) -> pathlib.Path: j2_env = ConfWriter.getJ2Environment() template = j2_env.get_template(template_name) - output = template.render(custom_conf_files=ConfWriter.getCustomConfFileStems(config), - app=config.app) - - output_path = config.getPackageDirectoryPath()/app_output_path + output = template.render( + custom_conf_files=ConfWriter.getCustomConfFileStems(config), app=config.app + ) + + output_path = config.getPackageDirectoryPath() / app_output_path output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, 'a') as f: - output = output.encode('utf-8', 'ignore').decode('utf-8') + with open(output_path, "a") as f: + output = output.encode("utf-8", "ignore").decode("utf-8") f.write(output) return output_path @staticmethod - def writeManifestFile(app_output_path:pathlib.Path, template_name : str, config: build, objects : list[CustomApp]) -> pathlib.Path: + def writeManifestFile( + app_output_path: pathlib.Path, + template_name: str, + config: build, + objects: list[CustomApp], + ) -> pathlib.Path: j2_env = ConfWriter.getJ2Environment() template = j2_env.get_template(template_name) - - output = template.render(objects=objects, app=config.app, currentDate=datetime.datetime.now(datetime.UTC).date().isoformat()) - - output_path = config.getPackageDirectoryPath()/app_output_path + + output = template.render( + objects=objects, + app=config.app, + currentDate=datetime.datetime.now(datetime.UTC).date().isoformat(), + ) + + output_path = config.getPackageDirectoryPath() / app_output_path output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, 'w') as f: - output = output.encode('utf-8', 'ignore').decode('utf-8') + with open(output_path, "w") as f: + output = output.encode("utf-8", "ignore").decode("utf-8") f.write(output) return output_path - - @staticmethod - def writeFileHeader(app_output_path:pathlib.Path, config: build) -> str: - #Do not output microseconds or +00:000 at the end of the datetime string - utc_time = datetime.datetime.now(datetime.UTC).replace(microsecond=0,tzinfo=None).isoformat() - - j2_env = Environment( - loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')), - trim_blocks=True) + def writeFileHeader(app_output_path: pathlib.Path, config: build) -> str: + # Do not output microseconds or +00:000 at the end of the datetime string + utc_time = ( + datetime.datetime.now(datetime.UTC) + .replace(microsecond=0, tzinfo=None) + .isoformat() + ) - template = j2_env.get_template('header.j2') - output = template.render(time=utc_time, author=' - '.join([config.app.author_name,config.app.author_company]), author_email=config.app.author_email) - - return output + j2_env = Environment( + loader=FileSystemLoader( + os.path.join(os.path.dirname(__file__), "templates") + ), + trim_blocks=True, + ) + template = j2_env.get_template("header.j2") + output = template.render( + time=utc_time, + author=" - ".join([config.app.author_name, config.app.author_company]), + author_email=config.app.author_email, + ) + return output @staticmethod - def writeXmlFile(app_output_path:pathlib.Path, template_name : str, config: build, objects : list[str]) -> None: - - + def writeXmlFile( + app_output_path: pathlib.Path, + template_name: str, + config: build, + objects: list[str], + ) -> None: j2_env = ConfWriter.getJ2Environment() template = j2_env.get_template(template_name) - + output = template.render(objects=objects, app=config.app) - - output_path = config.getPackageDirectoryPath()/app_output_path + + output_path = config.getPackageDirectoryPath() / app_output_path output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, 'a') as f: - output = output.encode('utf-8', 'ignore').decode('utf-8') + with open(output_path, "a") as f: + output = output.encode("utf-8", "ignore").decode("utf-8") f.write(output) - - #Ensure that the conf file we just generated/update is syntactically valid - ConfWriter.validateXmlFile(output_path) - + # Ensure that the conf file we just generated/update is syntactically valid + ConfWriter.validateXmlFile(output_path) @staticmethod - def writeDashboardFiles(config:build, dashboards:list[Dashboard])->set[pathlib.Path]: - written_files:set[pathlib.Path] = set() + def writeDashboardFiles( + config: build, dashboards: list[Dashboard] + ) -> set[pathlib.Path]: + written_files: set[pathlib.Path] = set() for dashboard in dashboards: output_file_path = dashboard.getOutputFilepathRelativeToAppRoot(config) # Check that the full output path does not exist so that we are not having an # name collision with a file in app_template - if (config.getPackageDirectoryPath()/output_file_path).exists(): - raise FileExistsError(f"ERROR: Overwriting Dashboard File {output_file_path}. Does this file exist in {config.getAppTemplatePath()} AND {config.path/'dashboards'}?") - + if (config.getPackageDirectoryPath() / output_file_path).exists(): + raise FileExistsError( + f"ERROR: Overwriting Dashboard File {output_file_path}. Does this file exist in {config.getAppTemplatePath()} AND {config.path / 'dashboards'}?" + ) + ConfWriter.writeXmlFileHeader(output_file_path, config) dashboard.writeDashboardFile(ConfWriter.getJ2Environment(), config) - ConfWriter.validateXmlFile(config.getPackageDirectoryPath()/output_file_path) + ConfWriter.validateXmlFile( + config.getPackageDirectoryPath() / output_file_path + ) written_files.add(output_file_path) return written_files - @staticmethod - def writeXmlFileHeader(app_output_path:pathlib.Path, config: build) -> None: - output = ConfWriter.writeFileHeader(app_output_path, config) + def writeXmlFileHeader(app_output_path: pathlib.Path, config: build) -> None: + output = ConfWriter.writeFileHeader(app_output_path, config) output_with_xml_comment = f"\n" - output_path = config.getPackageDirectoryPath()/app_output_path + output_path = config.getPackageDirectoryPath() / app_output_path output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, 'w') as f: - output_with_xml_comment = output_with_xml_comment.encode('utf-8', 'ignore').decode('utf-8') + with open(output_path, "w") as f: + output_with_xml_comment = output_with_xml_comment.encode( + "utf-8", "ignore" + ).decode("utf-8") f.write(output_with_xml_comment) - - # We INTENTIONALLY do not validate the comment we wrote to the header. This is because right now, - # the file is an empty XML document (besides the commented header). This means that it will FAIL validation + # We INTENTIONALLY do not validate the comment we wrote to the header. This is because right now, + # the file is an empty XML document (besides the commented header). This means that it will FAIL validation @staticmethod - def getJ2Environment()->Environment: + def getJ2Environment() -> Environment: j2_env = Environment( - loader=FileSystemLoader(os.path.join(os.path.dirname(__file__), 'templates')), + loader=FileSystemLoader( + os.path.join(os.path.dirname(__file__), "templates") + ), trim_blocks=True, - undefined=StrictUndefined) - j2_env.globals.update(objectListToNameList=SecurityContentObject.objectListToNameList) - - - j2_env.filters['custom_jinja2_enrichment_filter'] = ConfWriter.custom_jinja2_enrichment_filter - j2_env.filters['escapeNewlines'] = ConfWriter.escapeNewlines + undefined=StrictUndefined, + ) + j2_env.globals.update( + objectListToNameList=SecurityContentObject.objectListToNameList + ) + + j2_env.filters["custom_jinja2_enrichment_filter"] = ( + ConfWriter.custom_jinja2_enrichment_filter + ) + j2_env.filters["escapeNewlines"] = ConfWriter.escapeNewlines return j2_env @staticmethod - def writeConfFile(app_output_path:pathlib.Path, template_name : str, config: build, objects : Sequence[SecurityContentObject] | list[CustomApp]) -> pathlib.Path: - output_path = config.getPackageDirectoryPath()/app_output_path + def writeConfFile( + app_output_path: pathlib.Path, + template_name: str, + config: build, + objects: Sequence[SecurityContentObject] | list[CustomApp], + ) -> pathlib.Path: + output_path = config.getPackageDirectoryPath() / app_output_path j2_env = ConfWriter.getJ2Environment() - + template = j2_env.get_template(template_name) - outputs: list[str] = [] - for obj in objects: - try: - outputs.append(template.render(objects=[obj], app=config.app)) - except Exception as e: - raise Exception(f"Failed writing the following object to file:\n" - f"Name:{obj.name if not isinstance(obj, CustomApp) else obj.title}\n" - f"Type {type(obj)}: \n" - f"Output File: {app_output_path}\n" - f"Error: {str(e)}\n") - - output_path.parent.mkdir(parents=True, exist_ok=True) - with open(output_path, 'a') as f: - output = ''.join(outputs).encode('utf-8', 'ignore').decode('utf-8') - f.write(output) + + # The following code, which is commented out, serializes one object at a time. + # This is extremely useful from a debugging perspective, because sometimes when + # serializing a large number of objects, exceptions throw in Jinja2 templates can + # be quite hard to diagnose. We leave this code in for use in debugging workflows: + SERIALIZE_ONE_AT_A_TIME = False + if SERIALIZE_ONE_AT_A_TIME: + outputs: list[str] = [] + for obj in objects: + try: + outputs.append(template.render(objects=[obj], app=config.app)) + except Exception as e: + raise Exception( + f"Failed writing the following object to file:\n" + f"Name:{obj.name if not isinstance(obj, CustomApp) else obj.title}\n" + f"Type {type(obj)}: \n" + f"Output File: {app_output_path}\n" + f"Error: {str(e)}\n" + ) + + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "a") as f: + output = "".join(outputs).encode("utf-8", "ignore").decode("utf-8") + f.write(output) + else: + output = template.render(objects=objects, app=config.app) + + output_path.parent.mkdir(parents=True, exist_ok=True) + with open(output_path, "a") as f: + output = output.encode("utf-8", "ignore").decode("utf-8") + f.write(output) + return output_path - - + @staticmethod - def validateConfFile(path:pathlib.Path): + def validateConfFile(path: pathlib.Path): """Ensure that the conf file is valid. We will do this by reading back the conf using RawConfigParser to ensure that it does not throw any parsing errors. This is particularly relevant because newlines contained in string fields may break the formatting of the conf file if they have been incorrectly escaped with - the 'ConfWriter.escapeNewlines()' function. + the 'ConfWriter.escapeNewlines()' function. If a conf file failes validation, we will throw an exception @@ -309,7 +371,7 @@ def validateConfFile(path:pathlib.Path): """ return if path.suffix != ".conf": - #there may be some other files built, so just ignore them + # there may be some other files built, so just ignore them return try: _ = configparser.RawConfigParser().read(path) @@ -317,30 +379,35 @@ def validateConfFile(path:pathlib.Path): raise Exception(f"Failed to validate .conf file {str(path)}: {str(e)}") @staticmethod - def validateXmlFile(path:pathlib.Path): + def validateXmlFile(path: pathlib.Path): """Ensure that the XML file is valid XML. Args: path (pathlib.Path): path to the xml file to validate - """ - + """ + try: - with open(path, 'r') as xmlFile: + with open(path, "r") as xmlFile: _ = ET.fromstring(xmlFile.read()) except Exception as e: raise Exception(f"Failed to validate .xml file {str(path)}: {str(e)}") - @staticmethod - def validateManifestFile(path:pathlib.Path): + def validateManifestFile(path: pathlib.Path): """Ensure that the Manifest file is valid JSON. Args: path (pathlib.Path): path to the manifest JSON file to validate - """ + """ return try: - with open(path, 'r') as manifestFile: + with open(path, "r") as manifestFile: _ = json.load(manifestFile) except Exception as e: - raise Exception(f"Failed to validate .manifest file {str(path)} (Note that .manifest files should contain only valid JSON-formatted data): {str(e)}") + raise Exception( + f"Failed to validate .manifest file {str(path)} (Note that .manifest files should contain only valid JSON-formatted data): {str(e)}" + ) + except Exception as e: + raise Exception( + f"Failed to validate .manifest file {str(path)} (Note that .manifest files should contain only valid JSON-formatted data): {str(e)}" + ) diff --git a/contentctl/output/templates/transforms.j2 b/contentctl/output/templates/transforms.j2 index 58011189..e8ec8c4b 100644 --- a/contentctl/output/templates/transforms.j2 +++ b/contentctl/output/templates/transforms.j2 @@ -1,8 +1,8 @@ {% for lookup in objects %} [{{ lookup.name }}] -{% if lookup.filename is defined and lookup.filename != None %} -filename = {{ lookup.filename.name }} +{% if lookup.app_filename is defined and lookup.app_filename != None %} +filename = {{ lookup.app_filename.name }} {% else %} collection = {{ lookup.collection }} external_type = kvstore