From 239373a41cf5f703de4e0a06e620fb06894cb398 Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Wed, 17 Jan 2024 11:44:38 +0000 Subject: [PATCH 1/5] feat: App config schema --- .github/workflows/ci.yml | 48 ++-------------- development/app_config_schema.py | 55 +++++++++++++++++++ .../app-config-schema.json | 40 ++++++++++++++ poetry.lock | 16 +++++- pyproject.toml | 2 + tasks.py | 37 +++++++++++-- 6 files changed, 149 insertions(+), 49 deletions(-) create mode 100644 development/app_config_schema.py create mode 100644 nautobot_firewall_models/app-config-schema.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8b4dc5c..151bd7a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -93,7 +93,7 @@ jobs: uses: "networktocode/gh-action-setup-poetry-environment@v4" - name: "Linting: yamllint" run: "poetry run invoke yamllint" - pylint: + check-in-docker: needs: - "bandit" - "ruff" @@ -136,53 +136,13 @@ jobs: run: "cp development/creds.example.env development/creds.env" - name: "Linting: pylint" run: "poetry run invoke pylint" - check-migrations: - needs: - - "bandit" - - "ruff" - - "flake8" - - "poetry" - - "yamllint" - - "black" - runs-on: "ubuntu-22.04" - strategy: - fail-fast: true - matrix: - python-version: ["3.11"] - nautobot-version: ["2.0.0"] - env: - INVOKE_NAUTOBOT_FIREWALL_MODELS_PYTHON_VER: "${{ matrix.python-version }}" - INVOKE_NAUTOBOT_FIREWALL_MODELS_NAUTOBOT_VER: "${{ matrix.nautobot-version }}" - steps: - - name: "Check out repository code" - uses: "actions/checkout@v4" - - name: "Setup environment" - uses: "networktocode/gh-action-setup-poetry-environment@v4" - - name: "Set up Docker Buildx" - id: "buildx" - uses: "docker/setup-buildx-action@v3" - - name: "Build" - uses: "docker/build-push-action@v5" - with: - builder: "${{ steps.buildx.outputs.name }}" - context: "./" - push: false - load: true - tags: "${{ env.APP_NAME }}/nautobot:${{ matrix.nautobot-version }}-py${{ matrix.python-version }}" - file: "./development/Dockerfile" - cache-from: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}" - cache-to: "type=gha,scope=${{ matrix.nautobot-version }}-py${{ matrix.python-version }}" - build-args: | - NAUTOBOT_VER=${{ matrix.nautobot-version }} - PYTHON_VER=${{ matrix.python-version }} - - name: "Copy credentials" - run: "cp development/creds.example.env development/creds.env" + - name: "Checking: app-config-schema" + run: "poetry run invoke check-app-config-schema" - name: "Checking: migrations" run: "poetry run invoke check-migrations" unittest: needs: - - "pylint" - - "check-migrations" + - "check-in-docker" strategy: fail-fast: true matrix: diff --git a/development/app_config_schema.py b/development/app_config_schema.py new file mode 100644 index 0000000..8665a4c --- /dev/null +++ b/development/app_config_schema.py @@ -0,0 +1,55 @@ +"""App Config Schema Generator and Validator.""" +import json +from importlib import import_module +from os import getenv +from pathlib import Path +from urllib.parse import urlparse + +import jsonschema +import toml +from django.conf import settings +from to_json_schema.to_json_schema import SchemaBuilder + + +def _enrich_object_schema(schema, defaults, required): + schema["additionalProperties"] = False + for key, value in schema["properties"].items(): + if required and key in required: + value["required"] = True + default_value = defaults and defaults.get(key, None) + if value["type"] == "object" and "properties" in value: + _enrich_object_schema(value, default_value, None) + elif default_value is not None: + value["default"] = default_value + + +def _main(): + pyproject = toml.loads(Path("pyproject.toml").read_text()) + url = urlparse(pyproject["tool"]["poetry"]["repository"]) + _, owner, repository = url.path.split("/") + package_name = pyproject["tool"]["poetry"]["packages"][0]["include"] + app_config = settings.PLUGINS_CONFIG[package_name] # type: ignore + schema_path = Path(package_name) / "app-config-schema.json" + command = getenv("APP_CONFIG_SCHEMA_COMMAND", "") + if command == "generate": + schema = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": f"https://raw.githubusercontent.com/{owner}/{repository}/develop/{package_name}/app-config-schema.json", + "$comment": "TBD: Update $id, replace `develop` with the future release tag", + **SchemaBuilder().to_json_schema(app_config), # type: ignore + } + app_config = import_module(package_name).config + _enrich_object_schema(schema, app_config.default_settings, app_config.required_settings) + schema_path.write_text(json.dumps(schema, indent=4)) + print(f"\n==================\nGenerated schema:\n{schema_path}\nVerify the file manually and commit it.") + elif command == "validate": + schema = json.loads(schema_path.read_text()) + jsonschema.validate(app_config, schema) + print( + f"\n==================\nValidated configuration using the schema:\n{schema_path}\nConfiguration is valid." + ) + else: + raise RuntimeError(f"Unknown command: {command}") + + +_main() diff --git a/nautobot_firewall_models/app-config-schema.json b/nautobot_firewall_models/app-config-schema.json new file mode 100644 index 0000000..6c0e607 --- /dev/null +++ b/nautobot_firewall_models/app-config-schema.json @@ -0,0 +1,40 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://raw.githubusercontent.com/nautobot/nautobot-app-firewall-models/develop/nautobot_firewall_models/app-config-schema.json", + "$comment": "TBD: Update $id, replace `develop` with the future release tag", + "type": "object", + "properties": { + "capirca_os_map": { + "type": "object", + "properties": { + "cisco_ios": { + "type": "string", + "default": {} + }, + "arista_eos": { + "type": "string", + "default": {} + } + }, + "additionalProperties": false + }, + "capirca_remark_pass": { + "type": "boolean", + "default": true + }, + "allowed_status": { + "type": "array", + "items": { + "type": "string" + }, + "default": [ + "Active" + ] + }, + "protect_on_delete": { + "type": "boolean", + "default": true + } + }, + "additionalProperties": false +} diff --git a/poetry.lock b/poetry.lock index a97ca9d..b6c3159 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3196,6 +3196,20 @@ files = [ {file = "text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8"}, ] +[[package]] +name = "to-json-schema" +version = "1.0.1" +description = "" +optional = false +python-versions = "*" +files = [ + {file = "to_json_schema-1.0.1-py3-none-any.whl", hash = "sha256:5708663f1c81815e4ff01fce910ac32ee3964d0c6b3587fd4fff2e38d5c9aa7b"}, + {file = "to_json_schema-1.0.1.tar.gz", hash = "sha256:ec747bd5129256dd571105f66a7bc9a4546228cd5e5fbf5e06dc9776e255400e"}, +] + +[package.extras] +testing = ["pytest", "pytest-cov", "setuptools"] + [[package]] name = "toml" version = "0.10.2" @@ -3494,4 +3508,4 @@ all = [] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "c8733d31659d8bcacbac2217555d8775d605e6adfd5d593b56d6ec62fc0dc19e" +content-hash = "dd2d6d0e9a99224eebe95efed1cfcb9d3f99b033615164523abcafd0ac6955d8" diff --git a/pyproject.toml b/pyproject.toml index 95cb3e8..f8c0265 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,6 +53,8 @@ mkdocs-version-annotations = "1.0.0" mkdocstrings = "0.22.0" mkdocstrings-python = "1.5.2" towncrier = "~23.6.0" +to-json-schema = "*" +jsonschema = "*" [tool.poetry.extras] all = [ diff --git a/tasks.py b/tasks.py index 09095ab..33d9de1 100644 --- a/tasks.py +++ b/tasks.py @@ -151,15 +151,27 @@ def docker_compose(context, command, **kwargs): def run_command(context, command, **kwargs): """Wrapper to run a command locally or inside the nautobot container.""" if is_truthy(context.nautobot_firewall_models.local): + if "command_env" in kwargs: + kwargs["env"] = { + **kwargs.get("env", {}), + **kwargs.pop("command_env"), + } context.run(command, **kwargs) else: # Check if nautobot is running, no need to start another nautobot container to run a command docker_compose_status = "ps --services --filter status=running" results = docker_compose(context, docker_compose_status, hide="out") if "nautobot" in results.stdout: - compose_command = f"exec nautobot {command}" + compose_command = "exec" else: - compose_command = f"run --rm --entrypoint '{command}' nautobot" + compose_command = "run --rm --entrypoint=''" + + if "command_env" in kwargs: + command_env = kwargs.pop("command_env") + for key, value in command_env.items(): + compose_command += f' --env="{key}={value}"' + + compose_command += f" -- nautobot {command}" pty = kwargs.pop("pty", True) @@ -330,14 +342,15 @@ def logs(context, service="", follow=False, tail=0): # ACTIONS # ------------------------------------------------------------------------------ @task(help={"file": "Python file to execute"}) -def nbshell(context, file=""): +def nbshell(context, file="", env={}, plain=False): """Launch an interactive nbshell session.""" command = [ "nautobot-server", "nbshell", + "--plain" if plain else "", f"< '{file}'" if file else "", ] - run_command(context, " ".join(command), pty=not bool(file)) + run_command(context, " ".join(command), pty=not bool(file), command_env=env) @task @@ -801,8 +814,24 @@ def tests(context, failfast=False, keepdb=False, lint_only=False): pylint(context) print("Running mkdocs...") build_and_check_docs(context) + print("Checking app config schema...") + validate_app_config(context) if not lint_only: print("Running unit tests...") unittest(context, failfast=failfast, keepdb=keepdb) unittest_coverage(context) print("All tests have passed!") + + +@task +def generate_app_config_schema(context): + """Generate the app config schema based on the app config.""" + start(context, service="nautobot") + nbshell(context, file="development/app_config_schema.py", env={"APP_CONFIG_SCHEMA_COMMAND": "generate"}) + + +@task +def validate_app_config(context): + """Validate the app config based on the app config schema.""" + start(context, service="nautobot") + nbshell(context, plain=True, file="development/app_config_schema.py", env={"APP_CONFIG_SCHEMA_COMMAND": "validate"}) From 85f5dc6b3fdbae3e5a1ecd4154018017abc34cc3 Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Wed, 17 Jan 2024 11:47:59 +0000 Subject: [PATCH 2/5] chore: Add changelog fragment --- changes/213.added | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 changes/213.added diff --git a/changes/213.added b/changes/213.added new file mode 100644 index 0000000..733210d --- /dev/null +++ b/changes/213.added @@ -0,0 +1,3 @@ +Added `invoke generate-app-config-schema` command to generate a JSON schema for the App config. +Added `invoke validate-app-config` command to validate the App config against the schema. +Added App config JSON schema. From 92a3d93425d94edf39a74704017f0280d935768c Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Wed, 17 Jan 2024 11:50:11 +0000 Subject: [PATCH 3/5] fix: CI --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 151bd7a..64a8597 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,8 +136,8 @@ jobs: run: "cp development/creds.example.env development/creds.env" - name: "Linting: pylint" run: "poetry run invoke pylint" - - name: "Checking: app-config-schema" - run: "poetry run invoke check-app-config-schema" + - name: "Checking: App Config" + run: "poetry run invoke validate-app-config" - name: "Checking: migrations" run: "poetry run invoke check-migrations" unittest: From aada0d28b3d64b6c97fa31478ca7e7c19867941a Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Thu, 18 Jan 2024 11:43:38 +0000 Subject: [PATCH 4/5] fix: Better document generate invoke task --- development/app_config_schema.py | 13 ++++++++-- .../app-config-schema.json | 26 ++++++------------- tasks.py | 11 +++++++- 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/development/app_config_schema.py b/development/app_config_schema.py index 8665a4c..4700995 100644 --- a/development/app_config_schema.py +++ b/development/app_config_schema.py @@ -40,8 +40,17 @@ def _main(): } app_config = import_module(package_name).config _enrich_object_schema(schema, app_config.default_settings, app_config.required_settings) - schema_path.write_text(json.dumps(schema, indent=4)) - print(f"\n==================\nGenerated schema:\n{schema_path}\nVerify the file manually and commit it.") + schema_path.write_text(json.dumps(schema, indent=4) + "\n") + print(f"\n==================\nGenerated schema:\n\n{schema_path}\n") + print( + "WARNING: Review and edit the generated file before committing.\n" + "\n" + "Its content is inferred from:\n" + "\n" + "- The current configuration in `PLUGINS_CONFIG`\n" + "- `NautobotAppConfig.default_settings`\n" + "- `NautobotAppConfig.required_settings`" + ) elif command == "validate": schema = json.loads(schema_path.read_text()) jsonschema.validate(app_config, schema) diff --git a/nautobot_firewall_models/app-config-schema.json b/nautobot_firewall_models/app-config-schema.json index 6c0e607..e4a2de1 100644 --- a/nautobot_firewall_models/app-config-schema.json +++ b/nautobot_firewall_models/app-config-schema.json @@ -4,24 +4,6 @@ "$comment": "TBD: Update $id, replace `develop` with the future release tag", "type": "object", "properties": { - "capirca_os_map": { - "type": "object", - "properties": { - "cisco_ios": { - "type": "string", - "default": {} - }, - "arista_eos": { - "type": "string", - "default": {} - } - }, - "additionalProperties": false - }, - "capirca_remark_pass": { - "type": "boolean", - "default": true - }, "allowed_status": { "type": "array", "items": { @@ -31,6 +13,14 @@ "Active" ] }, + "capirca_remark_pass": { + "type": "boolean", + "default": true + }, + "capirca_os_map": { + "type": "object", + "default": {} + }, "protect_on_delete": { "type": "boolean", "default": true diff --git a/tasks.py b/tasks.py index 33d9de1..bfe5f2d 100644 --- a/tasks.py +++ b/tasks.py @@ -825,7 +825,16 @@ def tests(context, failfast=False, keepdb=False, lint_only=False): @task def generate_app_config_schema(context): - """Generate the app config schema based on the app config.""" + """Generate the app config schema from the current app config. + + WARNING: Review and edit the generated file before committing. + + Its content is inferred from: + + - The current configuration in `PLUGINS_CONFIG` + - `NautobotAppConfig.default_settings` + - `NautobotAppConfig.required_settings`""" + start(context, service="nautobot") nbshell(context, file="development/app_config_schema.py", env={"APP_CONFIG_SCHEMA_COMMAND": "generate"}) From 18bb0c003dff0f1de0c194fafd984ffeb74dc766 Mon Sep 17 00:00:00 2001 From: Jan Snasel Date: Thu, 18 Jan 2024 12:32:49 +0000 Subject: [PATCH 5/5] fix: Code formatting --- tasks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks.py b/tasks.py index bfe5f2d..3d6108e 100644 --- a/tasks.py +++ b/tasks.py @@ -833,8 +833,8 @@ def generate_app_config_schema(context): - The current configuration in `PLUGINS_CONFIG` - `NautobotAppConfig.default_settings` - - `NautobotAppConfig.required_settings`""" - + - `NautobotAppConfig.required_settings` + """ start(context, service="nautobot") nbshell(context, file="development/app_config_schema.py", env={"APP_CONFIG_SCHEMA_COMMAND": "generate"})