From 82c158665c167558acd5fed546f6a12bde5361a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Magnus=20B=C3=A4ck?= Date: Thu, 29 Jun 2023 15:08:04 +0200 Subject: [PATCH 1/5] Add diff_definitions.py This new script makes it easy to compare the current definitions of events and other types against a chosen base, e.g. the current state of the master branch or a previous edition. It produces diff commands that can be run to produce the wanted comparison. Since figuring out the most recent version of each type is a pretty common operation, a new versions module is introduced for this purpose. The existing find-latest-schemas.py script has been adapted to use the new module. --- diff_definitions.py | 61 +++++++++++++++++++++++++++ find-latest-schemas.py | 35 ++++++---------- test_versions.py | 93 ++++++++++++++++++++++++++++++++++++++++++ versions.py | 68 ++++++++++++++++++++++++++++++ 4 files changed, 235 insertions(+), 22 deletions(-) create mode 100755 diff_definitions.py create mode 100644 test_versions.py create mode 100644 versions.py diff --git a/diff_definitions.py b/diff_definitions.py new file mode 100755 index 00000000..7bec9b83 --- /dev/null +++ b/diff_definitions.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Axis Communications AB. +# For a full list of individual contributors, please see the commit history. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Context-sensitive diffing of Eiffel type definitions. Compares the +currently available types with the ones from a specified base and +prints diff commands. For example, if the current commit has added +v4.3.0 of ActT you'll get the following output: + +diff -u definitions/EiffelActivityTriggeredEvent/4.2.0.yml definitions/EiffelActivityTriggeredEvent/4.3.0.yml +diff -u schemas/EiffelActivityTriggeredEvent/4.2.0.json schemas/EiffelActivityTriggeredEvent/4.3.0.json + +By default, the base of the comparison is origin/master, but any commit +reference can be given as an argument. +""" + +import sys +from pathlib import Path + +import versions + + +def _main(): + base = "origin/master" + if len(sys.argv) > 2: + print(f"Usage: python {sys.argv[0]} [ base ]") + sys.exit(-1) + elif len(sys.argv) == 2: + base = sys.argv[1] + + earlier = versions.latest_in_gitref(base, ".", Path("definitions")) + current = versions.latest_in_dir(Path("definitions")) + for type, current_version in sorted(current.items()): + earlier_version = earlier.get(type) + if not earlier_version: + print(f"diff -u /dev/null definitions/{type}/{current_version}.yml") + print(f"diff -u /dev/null schemas/{type}/{current_version}.json") + elif earlier_version != current_version: + print( + f"diff -u definitions/{type}/{earlier_version}.yml definitions/{type}/{current_version}.yml" + ) + print( + f"diff -u schemas/{type}/{earlier_version}.json schemas/{type}/{current_version}.json" + ) + + +if __name__ == "__main__": + _main() diff --git a/find-latest-schemas.py b/find-latest-schemas.py index a35b9ad0..cb949dfc 100644 --- a/find-latest-schemas.py +++ b/find-latest-schemas.py @@ -1,8 +1,8 @@ -import os import sys +from pathlib import Path from shutil import copyfile -import semver +import versions """ Finds the latest versions of schemas found under , with @@ -16,28 +16,19 @@ def main(): if len(sys.argv) != 3: - print("Usage: python {} input_folder output_folder".format(sys.argv[0])) + print(f"Usage: python {sys.argv[0]} input_folder output_folder") sys.exit(-1) - input_folder = sys.argv[1] - output_folder = sys.argv[2] - - if not os.path.exists(output_folder): - os.makedirs(output_folder) - - latest_versions = [] - for base, _, files in os.walk(input_folder): - if len(files) != 0: - latest_version = "0.0.0" - for f in files: - latest_version = semver.max_ver(latest_version, os.path.splitext(f)[0]) - latest_versions.append(os.path.join(base, latest_version + ".json")) - - for f in latest_versions: - new_name = os.path.split(os.path.dirname(f))[1] + ".json" - output_file = os.path.join(output_folder, new_name) - copyfile(f, output_file) - print("{} => {}".format(f, output_file)) + input_folder = Path(sys.argv[1]) + output_folder = Path(sys.argv[2]) + + output_folder.mkdir(exist_ok=True) + + for type, version in versions.latest_in_dir(input_folder).items(): + input_file = input_folder / type / f"{version}.json" + output_file = output_folder / f"{type}.json" + copyfile(input_file, output_file) + print(f"{input_file} => {output_file}") if __name__ == "__main__": diff --git a/test_versions.py b/test_versions.py new file mode 100644 index 00000000..bb3bb5b5 --- /dev/null +++ b/test_versions.py @@ -0,0 +1,93 @@ +# Copyright 2024 Axis Communications AB. +# For a full list of individual contributors, please see the commit history. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import subprocess +from pathlib import Path + +import pytest +import semver + +import versions + + +class Git: + """Simple class for running Git commands in a given directory.""" + + def __init__(self, path: Path): + self.path = path + self.command("init") + # "git commit" requires that we identify ourselves. + self.command("config", "user.name", "John Doe") + self.command("config", "user.email", "john@example.com") + + def command(self, *args: str) -> None: + subprocess.check_call(["git"] + list(args), cwd=self.path) + + +@pytest.fixture +def tmp_git(tmp_path): + """Injects a Git instance rooted in a temporary directory.""" + yield Git(tmp_path) + + +def create_files(base_path: Path, *args: str) -> None: + for p in args: + fullpath = base_path / p + fullpath.parent.mkdir(parents=True, exist_ok=True) + fullpath.touch() + + +def test_latest_in_gitref(tmp_git): + # Create a bunch of files in the git, commit them, and tag that commit. + create_files( + tmp_git.path, + "subdir_c/6.0.0.json", + "definitions/subdir_a/1.0.0.json", + "definitions/subdir_a/2.0.0.json", + "definitions/subdir_b/3.0.0.json", + "definitions/subdir_b/4.0.0.json", + ) + tmp_git.command("add", "-A") + tmp_git.command("commit", "-m", "Initial set of files") + tmp_git.command("tag", "v1.0.0") + + # Add an additional file and delete one of the original files. + (tmp_git.path / "definitions/subdir_b/5.0.0.json").touch() + tmp_git.command("rm", "definitions/subdir_a/2.0.0.json") + tmp_git.command("add", "-A") + tmp_git.command("commit", "-m", "Make changes") + + # Make sure the results we get are consistent with the original + # contents of the git. + assert versions.latest_in_gitref("v1.0.0", tmp_git.path, "definitions") == { + "subdir_a": semver.VersionInfo.parse("2.0.0"), + "subdir_b": semver.VersionInfo.parse("4.0.0"), + } + + +def test_latest_in_dir(tmp_path): + create_files( + tmp_path, + "subdir_c/6.0.0.json", + "definitions/subdir_a/1.0.0.json", + "definitions/subdir_a/2.0.0.json", + "definitions/subdir_b/3.0.0.json", + "definitions/subdir_b/4.0.0.json", + ) + + assert versions.latest_in_dir(tmp_path / "definitions") == { + "subdir_a": semver.VersionInfo.parse("2.0.0"), + "subdir_b": semver.VersionInfo.parse("4.0.0"), + } diff --git a/versions.py b/versions.py new file mode 100644 index 00000000..8de71a7b --- /dev/null +++ b/versions.py @@ -0,0 +1,68 @@ +# Copyright 2024 Axis Communications AB. +# For a full list of individual contributors, please see the commit history. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The versions module contains functions for discovering definition files.""" + +import os +import subprocess +from pathlib import Path +from typing import Dict +from typing import Iterable + +import semver + + +def latest_in_gitref( + committish: str, gitdir: Path, subdir: Path +) -> Dict[str, semver.version.Version]: + """Lists the definition files found under a given subdirectory of a + git at a given point in time (described by a committish, e.g. a + SHA-1, tag, or branch reference) and returns a dict that maps each + typename (e.g. EiffelArtifactCreatedEvent) to the latest version found. + """ + return _latest_versions( + Path(line) + for line in ( + subprocess.check_output( + ["git", "ls-tree", "-r", "--name-only", committish, "--", subdir], + cwd=gitdir, + ) + .decode("utf-8") + .splitlines() + ) + ) + + +def latest_in_dir(path: Path) -> Dict[str, semver.version.Version]: + """Inspects the definition files found under a given path and returns + a dict that maps each typename (e.g. EiffelArtifactCreatedEvent) to + its latest version found. + """ + return _latest_versions( + Path(current) / f for current, _, files in os.walk(path) for f in files + ) + + +def _latest_versions(paths: Iterable[Path]) -> Dict[str, semver.version.Version]: + """Given a list of foo//. pathnames, returns + a dict mapping typenames to the most recent version of that type. + """ + result = {} + for path in paths: + type = path.parent.name + this_version = semver.VersionInfo.parse(Path(path.name).stem) + if type not in result or result[type] < this_version: + result[type] = this_version + return result From a0628c12df07b4c993ee4649b4256f1459ea4297 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Magnus=20B=C3=A4ck?= Date: Tue, 9 Jan 2024 17:20:27 +0100 Subject: [PATCH 2/5] Add copyright notice to find-latest-schemas.py --- find-latest-schemas.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/find-latest-schemas.py b/find-latest-schemas.py index cb949dfc..3f969d14 100644 --- a/find-latest-schemas.py +++ b/find-latest-schemas.py @@ -1,3 +1,18 @@ +# Copyright 2016-2024 Ericsson AB and others. +# For a full list of individual contributors, please see the commit history. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import sys from pathlib import Path from shutil import copyfile From e0d048628b83edd37e2a96410b79a2b9edec091d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Magnus=20B=C3=A4ck?= Date: Wed, 10 Jan 2024 16:41:50 +0100 Subject: [PATCH 3/5] diff_definitions.py: Rephrase file docstring --- diff_definitions.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/diff_definitions.py b/diff_definitions.py index 7bec9b83..2b90e6ac 100755 --- a/diff_definitions.py +++ b/diff_definitions.py @@ -16,9 +16,10 @@ # limitations under the License. """Context-sensitive diffing of Eiffel type definitions. Compares the -currently available types with the ones from a specified base and -prints diff commands. For example, if the current commit has added -v4.3.0 of ActT you'll get the following output: +current workspace's latest type versions with the ones from a specified +base, and prints diff commands. For example, if the current commit has +added v4.3.0 of ActT and the given base had v4.2.0 as its latest version, +you'll get the following output: diff -u definitions/EiffelActivityTriggeredEvent/4.2.0.yml definitions/EiffelActivityTriggeredEvent/4.3.0.yml diff -u schemas/EiffelActivityTriggeredEvent/4.2.0.json schemas/EiffelActivityTriggeredEvent/4.3.0.json From 1fa321115a0ce3fb90a95e0b30d720e08baa8300 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Magnus=20B=C3=A4ck?= Date: Wed, 10 Jan 2024 16:46:22 +0100 Subject: [PATCH 4/5] diff_definitions.py: Use base/workspace terminology instead of earlier/current --- diff_definitions.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/diff_definitions.py b/diff_definitions.py index 2b90e6ac..59079883 100755 --- a/diff_definitions.py +++ b/diff_definitions.py @@ -42,19 +42,19 @@ def _main(): elif len(sys.argv) == 2: base = sys.argv[1] - earlier = versions.latest_in_gitref(base, ".", Path("definitions")) - current = versions.latest_in_dir(Path("definitions")) - for type, current_version in sorted(current.items()): - earlier_version = earlier.get(type) - if not earlier_version: - print(f"diff -u /dev/null definitions/{type}/{current_version}.yml") - print(f"diff -u /dev/null schemas/{type}/{current_version}.json") - elif earlier_version != current_version: + base_defs = versions.latest_in_gitref(base, ".", Path("definitions")) + workspace_defs = versions.latest_in_dir(Path("definitions")) + for type, workspace_version in sorted(workspace_defs.items()): + base_version = base_defs.get(type) + if not base_version: + print(f"diff -u /dev/null definitions/{type}/{workspace_version}.yml") + print(f"diff -u /dev/null schemas/{type}/{workspace_version}.json") + elif base_version != workspace_version: print( - f"diff -u definitions/{type}/{earlier_version}.yml definitions/{type}/{current_version}.yml" + f"diff -u definitions/{type}/{base_version}.yml definitions/{type}/{workspace_version}.yml" ) print( - f"diff -u schemas/{type}/{earlier_version}.json schemas/{type}/{current_version}.json" + f"diff -u schemas/{type}/{base_version}.json schemas/{type}/{workspace_version}.json" ) From 5263505a3bded89223dc198cfc6e74392f2aad50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Magnus=20B=C3=A4ck?= Date: Fri, 12 Jan 2024 20:13:46 +0100 Subject: [PATCH 5/5] Ignore non-YAML files --- test_versions.py | 24 +++++++++++++----------- versions.py | 6 +++++- 2 files changed, 18 insertions(+), 12 deletions(-) diff --git a/test_versions.py b/test_versions.py index bb3bb5b5..ec263f82 100644 --- a/test_versions.py +++ b/test_versions.py @@ -54,18 +54,19 @@ def test_latest_in_gitref(tmp_git): create_files( tmp_git.path, "subdir_c/6.0.0.json", - "definitions/subdir_a/1.0.0.json", - "definitions/subdir_a/2.0.0.json", - "definitions/subdir_b/3.0.0.json", - "definitions/subdir_b/4.0.0.json", + "definitions/subdir_a/1.0.0.yml", + "definitions/subdir_a/2.0.0.yml", + "definitions/subdir_a/3.0.0.othersuffix", + "definitions/subdir_b/3.0.0.yml", + "definitions/subdir_b/4.0.0.yml", ) tmp_git.command("add", "-A") tmp_git.command("commit", "-m", "Initial set of files") tmp_git.command("tag", "v1.0.0") # Add an additional file and delete one of the original files. - (tmp_git.path / "definitions/subdir_b/5.0.0.json").touch() - tmp_git.command("rm", "definitions/subdir_a/2.0.0.json") + (tmp_git.path / "definitions/subdir_b/5.0.0.yml").touch() + tmp_git.command("rm", "definitions/subdir_a/2.0.0.yml") tmp_git.command("add", "-A") tmp_git.command("commit", "-m", "Make changes") @@ -80,11 +81,12 @@ def test_latest_in_gitref(tmp_git): def test_latest_in_dir(tmp_path): create_files( tmp_path, - "subdir_c/6.0.0.json", - "definitions/subdir_a/1.0.0.json", - "definitions/subdir_a/2.0.0.json", - "definitions/subdir_b/3.0.0.json", - "definitions/subdir_b/4.0.0.json", + "subdir_c/6.0.0.yml", + "definitions/subdir_a/1.0.0.yml", + "definitions/subdir_a/2.0.0.yml", + "definitions/subdir_a/3.0.0.othersuffix", + "definitions/subdir_b/3.0.0.yml", + "definitions/subdir_b/4.0.0.yml", ) assert versions.latest_in_dir(tmp_path / "definitions") == { diff --git a/versions.py b/versions.py index 8de71a7b..f250e01d 100644 --- a/versions.py +++ b/versions.py @@ -42,6 +42,7 @@ def latest_in_gitref( .decode("utf-8") .splitlines() ) + if Path(line).suffix == ".yml" ) @@ -51,7 +52,10 @@ def latest_in_dir(path: Path) -> Dict[str, semver.version.Version]: its latest version found. """ return _latest_versions( - Path(current) / f for current, _, files in os.walk(path) for f in files + Path(current) / f + for current, _, files in os.walk(path) + for f in files + if Path(f).suffix == ".yml" )