From 3a43cd2fb67f3929bf35ba8a507a81b86b31c4eb Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Tue, 16 Jul 2024 17:23:48 +0300 Subject: [PATCH 01/36] added gitub action --- .github/workflows/sync_native_dependency.yml | 31 ++++++++++++++++++++ docker/py3-native/pyproject.toml | 2 +- 2 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/sync_native_dependency.yml diff --git a/.github/workflows/sync_native_dependency.yml b/.github/workflows/sync_native_dependency.yml new file mode 100644 index 0000000000000..777419221894e --- /dev/null +++ b/.github/workflows/sync_native_dependency.yml @@ -0,0 +1,31 @@ +name: Compare Dependency Constraints +on: + pull_request: + branches: + - master +jobs: + compare_dependency_constraints_script: + runs-on: ubuntu-latest + if: github.repository == 'demisto/dockerFiles' + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v41 + - name: Set up Python 3.11 + uses: actions/setup-python@v3 + with: + python-version: "3.11" + - name: Run compare_dependency_constraints Script. + env: + CHANGED_FILES: "${{ steps.changed-files.outputs.all_changed_files }}" + run: | + echo "==== $(date): Starting environment setup... ====" + python -m pip install --upgrade pip + pwd + cd "$GITHUB_WORKSPACE/utils/" + echo "==== $(date): Running validator... ====" + echo "==== $(date): Running chceking Native dependency ====" + python ./compare_dependency_constraints.py \ No newline at end of file diff --git a/docker/py3-native/pyproject.toml b/docker/py3-native/pyproject.toml index fd9ea419ee476..f402f9c4399cd 100644 --- a/docker/py3-native/pyproject.toml +++ b/docker/py3-native/pyproject.toml @@ -45,7 +45,7 @@ parse-emails = "*" slack-sdk = {extras = ["optional"], version = "*"} pyjwt = {extras = ["crypto"],version = "*"} google-api-python-client = "*" -exchangelib = ">=5.4.1" +exchangelib = ">=5.4.0" requests-oauthlib = "*" oauthlib = {extras = ["signedtoken"], version = "*"} requests-ntlm = "==1.1.0" From c0ae7c17619193762d1eefbcdb43a84bb8722aa8 Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Tue, 16 Jul 2024 17:28:08 +0300 Subject: [PATCH 02/36] pipenv --- .github/workflows/sync_native_dependency.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/sync_native_dependency.yml b/.github/workflows/sync_native_dependency.yml index 777419221894e..dae785630ab66 100644 --- a/.github/workflows/sync_native_dependency.yml +++ b/.github/workflows/sync_native_dependency.yml @@ -24,6 +24,8 @@ jobs: run: | echo "==== $(date): Starting environment setup... ====" python -m pip install --upgrade pip + pipenv install + pipenv shell pwd cd "$GITHUB_WORKSPACE/utils/" echo "==== $(date): Running validator... ====" From 82907860d67c363087463c757c054993656c3c87 Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Tue, 16 Jul 2024 17:33:02 +0300 Subject: [PATCH 03/36] pipenv --- .github/workflows/sync_native_dependency.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/sync_native_dependency.yml b/.github/workflows/sync_native_dependency.yml index dae785630ab66..a993660cd5d03 100644 --- a/.github/workflows/sync_native_dependency.yml +++ b/.github/workflows/sync_native_dependency.yml @@ -18,15 +18,16 @@ jobs: uses: actions/setup-python@v3 with: python-version: "3.11" + - name: Install dependencies with pipenv + run: | + python -m pip install --upgrade pip + pip install pipenv + pipenv install - name: Run compare_dependency_constraints Script. env: CHANGED_FILES: "${{ steps.changed-files.outputs.all_changed_files }}" run: | echo "==== $(date): Starting environment setup... ====" - python -m pip install --upgrade pip - pipenv install - pipenv shell - pwd cd "$GITHUB_WORKSPACE/utils/" echo "==== $(date): Running validator... ====" echo "==== $(date): Running chceking Native dependency ====" From 69aee8eaa7b2ca7016a1ae5b264834fd14ac7c8f Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Tue, 16 Jul 2024 17:35:14 +0300 Subject: [PATCH 04/36] pipenv --- .github/workflows/sync_native_dependency.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sync_native_dependency.yml b/.github/workflows/sync_native_dependency.yml index a993660cd5d03..0f80cb0383854 100644 --- a/.github/workflows/sync_native_dependency.yml +++ b/.github/workflows/sync_native_dependency.yml @@ -23,6 +23,7 @@ jobs: python -m pip install --upgrade pip pip install pipenv pipenv install + pipenv shell - name: Run compare_dependency_constraints Script. env: CHANGED_FILES: "${{ steps.changed-files.outputs.all_changed_files }}" From 2c249f87c103043b124ed695af669cb1a3875ddd Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Tue, 16 Jul 2024 17:36:19 +0300 Subject: [PATCH 05/36] pipenv --- .github/workflows/sync_native_dependency.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync_native_dependency.yml b/.github/workflows/sync_native_dependency.yml index 0f80cb0383854..f73a5cc7ee9c6 100644 --- a/.github/workflows/sync_native_dependency.yml +++ b/.github/workflows/sync_native_dependency.yml @@ -32,4 +32,4 @@ jobs: cd "$GITHUB_WORKSPACE/utils/" echo "==== $(date): Running validator... ====" echo "==== $(date): Running chceking Native dependency ====" - python ./compare_dependency_constraints.py \ No newline at end of file + pipenv run ./compare_dependency_constraints.py \ No newline at end of file From debe4a95633c96f218d7c2ef2aa1087c14a1d182 Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Tue, 16 Jul 2024 17:38:16 +0300 Subject: [PATCH 06/36] remove pipenv shell --- .github/workflows/sync_native_dependency.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/sync_native_dependency.yml b/.github/workflows/sync_native_dependency.yml index f73a5cc7ee9c6..fad327cc02a7e 100644 --- a/.github/workflows/sync_native_dependency.yml +++ b/.github/workflows/sync_native_dependency.yml @@ -23,7 +23,6 @@ jobs: python -m pip install --upgrade pip pip install pipenv pipenv install - pipenv shell - name: Run compare_dependency_constraints Script. env: CHANGED_FILES: "${{ steps.changed-files.outputs.all_changed_files }}" From 4956cbba66171527c2aa2d70ddbc6a102656b877 Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Tue, 16 Jul 2024 17:40:16 +0300 Subject: [PATCH 07/36] pipenv install --- .github/workflows/sync_native_dependency.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/sync_native_dependency.yml b/.github/workflows/sync_native_dependency.yml index fad327cc02a7e..1fff40a18fe45 100644 --- a/.github/workflows/sync_native_dependency.yml +++ b/.github/workflows/sync_native_dependency.yml @@ -22,13 +22,12 @@ jobs: run: | python -m pip install --upgrade pip pip install pipenv - pipenv install - name: Run compare_dependency_constraints Script. env: CHANGED_FILES: "${{ steps.changed-files.outputs.all_changed_files }}" run: | echo "==== $(date): Starting environment setup... ====" cd "$GITHUB_WORKSPACE/utils/" - echo "==== $(date): Running validator... ====" + pipenv install echo "==== $(date): Running chceking Native dependency ====" pipenv run ./compare_dependency_constraints.py \ No newline at end of file From df88ee61e9aa1c335abb67c8897bb95e9d91f8cf Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Tue, 16 Jul 2024 17:41:38 +0300 Subject: [PATCH 08/36] path --- .github/workflows/sync_native_dependency.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/sync_native_dependency.yml b/.github/workflows/sync_native_dependency.yml index 1fff40a18fe45..08b28c767df26 100644 --- a/.github/workflows/sync_native_dependency.yml +++ b/.github/workflows/sync_native_dependency.yml @@ -9,8 +9,6 @@ jobs: if: github.repository == 'demisto/dockerFiles' steps: - uses: actions/checkout@v3 - with: - fetch-depth: 0 - name: Get changed files id: changed-files uses: tj-actions/changed-files@v41 @@ -30,4 +28,4 @@ jobs: cd "$GITHUB_WORKSPACE/utils/" pipenv install echo "==== $(date): Running chceking Native dependency ====" - pipenv run ./compare_dependency_constraints.py \ No newline at end of file + pipenv run compare_dependency_constraints.py \ No newline at end of file From 76fba504e877ab88133e12b48c4e9c306059f453 Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Tue, 16 Jul 2024 17:47:35 +0300 Subject: [PATCH 09/36] run python --- .github/workflows/sync_native_dependency.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync_native_dependency.yml b/.github/workflows/sync_native_dependency.yml index 08b28c767df26..eb7f0f2ea7ffb 100644 --- a/.github/workflows/sync_native_dependency.yml +++ b/.github/workflows/sync_native_dependency.yml @@ -28,4 +28,4 @@ jobs: cd "$GITHUB_WORKSPACE/utils/" pipenv install echo "==== $(date): Running chceking Native dependency ====" - pipenv run compare_dependency_constraints.py \ No newline at end of file + pipenv run python compare_dependency_constraints.py \ No newline at end of file From 7faa2e7af5b8ef7f406bddee8131e787e41cf804 Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Tue, 16 Jul 2024 17:49:26 +0300 Subject: [PATCH 10/36] pipenv install --- .github/workflows/sync_native_dependency.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sync_native_dependency.yml b/.github/workflows/sync_native_dependency.yml index eb7f0f2ea7ffb..ff62d3d161845 100644 --- a/.github/workflows/sync_native_dependency.yml +++ b/.github/workflows/sync_native_dependency.yml @@ -20,12 +20,12 @@ jobs: run: | python -m pip install --upgrade pip pip install pipenv + pipenv install - name: Run compare_dependency_constraints Script. env: CHANGED_FILES: "${{ steps.changed-files.outputs.all_changed_files }}" run: | echo "==== $(date): Starting environment setup... ====" cd "$GITHUB_WORKSPACE/utils/" - pipenv install echo "==== $(date): Running chceking Native dependency ====" pipenv run python compare_dependency_constraints.py \ No newline at end of file From ce78770583e408bda9d68dd7c8a839c341b75f77 Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Tue, 16 Jul 2024 17:51:27 +0300 Subject: [PATCH 11/36] remove changed-files --- .github/workflows/sync_native_dependency.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/sync_native_dependency.yml b/.github/workflows/sync_native_dependency.yml index ff62d3d161845..559d8e4ac03da 100644 --- a/.github/workflows/sync_native_dependency.yml +++ b/.github/workflows/sync_native_dependency.yml @@ -9,9 +9,6 @@ jobs: if: github.repository == 'demisto/dockerFiles' steps: - uses: actions/checkout@v3 - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v41 - name: Set up Python 3.11 uses: actions/setup-python@v3 with: @@ -22,8 +19,6 @@ jobs: pip install pipenv pipenv install - name: Run compare_dependency_constraints Script. - env: - CHANGED_FILES: "${{ steps.changed-files.outputs.all_changed_files }}" run: | echo "==== $(date): Starting environment setup... ====" cd "$GITHUB_WORKSPACE/utils/" From 0b0e2dfb2275613f1ff0a49c3a66d687d6be8ee8 Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Tue, 16 Jul 2024 17:57:13 +0300 Subject: [PATCH 12/36] error --- utils/compare_dependency_constraints.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/utils/compare_dependency_constraints.py b/utils/compare_dependency_constraints.py index 3e43b2b7946f8..45ac9320c325c 100644 --- a/utils/compare_dependency_constraints.py +++ b/utils/compare_dependency_constraints.py @@ -95,6 +95,10 @@ def compare_constraints(images_contained_in_native: list[str]): for discrepancy in discrepancies: print(str(discrepancy)) + print( # noqa: T201 + f"::error file={discrepancy.in_image},line=1,endLine=1,title=Error" + ) + def load_native_image_conf() -> list[str]: From a73f7d5dc6c5c013fc9fe462b8c15ea592016317 Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Tue, 16 Jul 2024 18:04:09 +0300 Subject: [PATCH 13/36] return 1/0 --- utils/compare_dependency_constraints.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/utils/compare_dependency_constraints.py b/utils/compare_dependency_constraints.py index 45ac9320c325c..77c01f3f02d2c 100644 --- a/utils/compare_dependency_constraints.py +++ b/utils/compare_dependency_constraints.py @@ -3,6 +3,7 @@ from typing import Any, NamedTuple import requests import toml +import sys DOCKER_FOLDER = Path(__file__).parent.parent / "docker" NATIVE_IMAGE = "py3-native" @@ -98,7 +99,9 @@ def compare_constraints(images_contained_in_native: list[str]): print( # noqa: T201 f"::error file={discrepancy.in_image},line=1,endLine=1,title=Error" ) - + if discrepancies: + return 1 + return 0 def load_native_image_conf() -> list[str]: @@ -110,4 +113,4 @@ def load_native_image_conf() -> list[str]: )["native_images"]["native:candidate"]["supported_docker_images"] -compare_constraints(load_native_image_conf()) +sys.exit(compare_constraints(load_native_image_conf())) From 3ac5966cc32325e3467ea19c6e74c54726b2d246 Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Tue, 16 Jul 2024 18:07:26 +0300 Subject: [PATCH 14/36] main --- utils/compare_dependency_constraints.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/utils/compare_dependency_constraints.py b/utils/compare_dependency_constraints.py index 77c01f3f02d2c..db121bc985051 100644 --- a/utils/compare_dependency_constraints.py +++ b/utils/compare_dependency_constraints.py @@ -96,9 +96,6 @@ def compare_constraints(images_contained_in_native: list[str]): for discrepancy in discrepancies: print(str(discrepancy)) - print( # noqa: T201 - f"::error file={discrepancy.in_image},line=1,endLine=1,title=Error" - ) if discrepancies: return 1 return 0 @@ -113,4 +110,5 @@ def load_native_image_conf() -> list[str]: )["native_images"]["native:candidate"]["supported_docker_images"] -sys.exit(compare_constraints(load_native_image_conf())) +if __name__ == '__main__': + sys.exit(compare_constraints(load_native_image_conf())) From ce86ffb5c383838aefc89d19920a97bb8da815b0 Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Tue, 16 Jul 2024 18:10:40 +0300 Subject: [PATCH 15/36] no return --- utils/compare_dependency_constraints.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/compare_dependency_constraints.py b/utils/compare_dependency_constraints.py index db121bc985051..0d8cb28bf793c 100644 --- a/utils/compare_dependency_constraints.py +++ b/utils/compare_dependency_constraints.py @@ -96,9 +96,9 @@ def compare_constraints(images_contained_in_native: list[str]): for discrepancy in discrepancies: print(str(discrepancy)) - if discrepancies: - return 1 - return 0 + # if discrepancies: + # return 1 + # return 0 def load_native_image_conf() -> list[str]: From e059f9fa4731b46ccc0575c3e63d957fb05308aa Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Tue, 16 Jul 2024 18:20:39 +0300 Subject: [PATCH 16/36] return --- utils/compare_dependency_constraints.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/utils/compare_dependency_constraints.py b/utils/compare_dependency_constraints.py index 0d8cb28bf793c..1fa87fa2f1a0a 100644 --- a/utils/compare_dependency_constraints.py +++ b/utils/compare_dependency_constraints.py @@ -94,11 +94,10 @@ def compare_constraints(images_contained_in_native: list[str]): ) ) - for discrepancy in discrepancies: - print(str(discrepancy)) - # if discrepancies: - # return 1 - # return 0 + for discrepancy in discrepancies: + print(str(discrepancy)) + + return int(bool(discrepancies)) def load_native_image_conf() -> list[str]: From b78da47cc9178fc82dddb46f01ee9b06a178fb52 Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Tue, 16 Jul 2024 18:25:39 +0300 Subject: [PATCH 17/36] print --- utils/compare_dependency_constraints.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/utils/compare_dependency_constraints.py b/utils/compare_dependency_constraints.py index 1fa87fa2f1a0a..b50518bffc12e 100644 --- a/utils/compare_dependency_constraints.py +++ b/utils/compare_dependency_constraints.py @@ -96,7 +96,9 @@ def compare_constraints(images_contained_in_native: list[str]): for discrepancy in discrepancies: print(str(discrepancy)) - + print( # noqa: T201 + f"::error file=docker/{discrepancy.image}/Dockerfile,line=1,endLine=1,title=Native Image Discrepancy::{discrepancy}" + ) return int(bool(discrepancies)) @@ -109,5 +111,5 @@ def load_native_image_conf() -> list[str]: )["native_images"]["native:candidate"]["supported_docker_images"] -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(compare_constraints(load_native_image_conf())) From 9e87de4b9a5674ffaa8a2e3aac2f9af9f5a5d494 Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Tue, 16 Jul 2024 18:28:23 +0300 Subject: [PATCH 18/36] fix --- utils/compare_dependency_constraints.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/utils/compare_dependency_constraints.py b/utils/compare_dependency_constraints.py index b50518bffc12e..8bf6764e2a90b 100644 --- a/utils/compare_dependency_constraints.py +++ b/utils/compare_dependency_constraints.py @@ -60,9 +60,8 @@ def compare_constraints(images_contained_in_native: list[str]): | parse_constraints(NATIVE_IMAGE) ) native_constraint_keys = set(native_constraints.keys()) - + discrepancies: list[Discrepancy] = [] for image in images_contained_in_native: - discrepancies: list[Discrepancy] = [] constraints = parse_constraints(image) constraint_keys = set(constraints.keys()) @@ -96,7 +95,7 @@ def compare_constraints(images_contained_in_native: list[str]): for discrepancy in discrepancies: print(str(discrepancy)) - print( # noqa: T201 + print( f"::error file=docker/{discrepancy.image}/Dockerfile,line=1,endLine=1,title=Native Image Discrepancy::{discrepancy}" ) return int(bool(discrepancies)) From 39dd6acd87a42e39db31ba05fbec062ef0d9afcb Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Tue, 16 Jul 2024 22:32:28 +0300 Subject: [PATCH 19/36] line number --- utils/compare_dependency_constraints.py | 26 ++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/utils/compare_dependency_constraints.py b/utils/compare_dependency_constraints.py index 8bf6764e2a90b..49075b372858e 100644 --- a/utils/compare_dependency_constraints.py +++ b/utils/compare_dependency_constraints.py @@ -52,6 +52,29 @@ class Discrepancy(NamedTuple): def __str__(self) -> str: return f"{self.dependency}: {self.in_image or 'missing'} in {self.image}, {self.in_native or 'missing'} in native" +def find_library_line_number(lib_name: str, file_path: Path): + """ + Searches for a library in the pyproject.toml or Pipfile file and returns the line number where it is found. + + Parameters: + - lib_name: The name of the library to search for. + - file_path: The directory containing the pyproject.toml or Pipfile. + + Returns: + - The line number containing the library name, or None if the library is not found. + """ + file_pyproject_path = file_path / "pyproject.toml" + if file_pyproject_path.exists(): + file_path = file_pyproject_path + else: + file_path = Path(file_path) / "Pipfile" + + with file_path.open('r') as file: + for line_number, line in enumerate(file, start=1): # Start counting from line 1 + if lib_name in line: + return line_number + + return None def compare_constraints(images_contained_in_native: list[str]): native_constraints = ( @@ -94,9 +117,10 @@ def compare_constraints(images_contained_in_native: list[str]): ) for discrepancy in discrepancies: + line_number = find_library_line_number(discrepancy.dependency, Path(f'docker/{discrepancy.image}')) print(str(discrepancy)) print( - f"::error file=docker/{discrepancy.image}/Dockerfile,line=1,endLine=1,title=Native Image Discrepancy::{discrepancy}" + f"::error file=docker/{discrepancy.image}/Dockerfile,line={line_number},endLine=1,title=Native Image Discrepancy::{discrepancy}" ) return int(bool(discrepancies)) From 942f0d100e9823996ed88acde80e68c0d3771581 Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Tue, 16 Jul 2024 22:34:26 +0300 Subject: [PATCH 20/36] cd --- .github/workflows/sync_native_dependency.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sync_native_dependency.yml b/.github/workflows/sync_native_dependency.yml index 559d8e4ac03da..d0737fbadfc5d 100644 --- a/.github/workflows/sync_native_dependency.yml +++ b/.github/workflows/sync_native_dependency.yml @@ -20,7 +20,8 @@ jobs: pipenv install - name: Run compare_dependency_constraints Script. run: | + ls echo "==== $(date): Starting environment setup... ====" - cd "$GITHUB_WORKSPACE/utils/" + cd "$GITHUB_WORKSPACE" echo "==== $(date): Running chceking Native dependency ====" - pipenv run python compare_dependency_constraints.py \ No newline at end of file + pipenv run python utils/compare_dependency_constraints.py \ No newline at end of file From 015381470e7b571dd90df92c364ae0fdb9e1aaaf Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Tue, 16 Jul 2024 22:45:51 +0300 Subject: [PATCH 21/36] print --- utils/compare_dependency_constraints.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/utils/compare_dependency_constraints.py b/utils/compare_dependency_constraints.py index 49075b372858e..b44625ec0af03 100644 --- a/utils/compare_dependency_constraints.py +++ b/utils/compare_dependency_constraints.py @@ -63,12 +63,6 @@ def find_library_line_number(lib_name: str, file_path: Path): Returns: - The line number containing the library name, or None if the library is not found. """ - file_pyproject_path = file_path / "pyproject.toml" - if file_pyproject_path.exists(): - file_path = file_pyproject_path - else: - file_path = Path(file_path) / "Pipfile" - with file_path.open('r') as file: for line_number, line in enumerate(file, start=1): # Start counting from line 1 if lib_name in line: @@ -117,10 +111,15 @@ def compare_constraints(images_contained_in_native: list[str]): ) for discrepancy in discrepancies: - line_number = find_library_line_number(discrepancy.dependency, Path(f'docker/{discrepancy.image}')) - print(str(discrepancy)) + file_pyproject_path = DOCKER_FOLDER / f"{discrepancy.image}/pyproject.toml" + if file_pyproject_path.exists(): + file_path = file_pyproject_path + else: + file_path = DOCKER_FOLDER / f"{discrepancy.image}/Pipfile" + line_number = find_library_line_number(discrepancy.dependency, file_path) + # print(str(discrepancy)) print( - f"::error file=docker/{discrepancy.image}/Dockerfile,line={line_number},endLine=1,title=Native Image Discrepancy::{discrepancy}" + f"::error file={file_path},line={line_number},endLine=1,title=Native Image Discrepancy::{discrepancy}" ) return int(bool(discrepancies)) From 31a9ec30a46ca5af5a5361657bbb8a34b7b4bb06 Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Tue, 16 Jul 2024 22:49:50 +0300 Subject: [PATCH 22/36] line_number --- utils/compare_dependency_constraints.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/compare_dependency_constraints.py b/utils/compare_dependency_constraints.py index b44625ec0af03..a7a533391d306 100644 --- a/utils/compare_dependency_constraints.py +++ b/utils/compare_dependency_constraints.py @@ -117,9 +117,9 @@ def compare_constraints(images_contained_in_native: list[str]): else: file_path = DOCKER_FOLDER / f"{discrepancy.image}/Pipfile" line_number = find_library_line_number(discrepancy.dependency, file_path) - # print(str(discrepancy)) + print( - f"::error file={file_path},line={line_number},endLine=1,title=Native Image Discrepancy::{discrepancy}" + f"::error file={file_path},line={line_number},endLine={line_number},title=Native Image Discrepancy::{discrepancy}" ) return int(bool(discrepancies)) From ed62dfb966a5d9e18355a107db386effafc56d22 Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Tue, 16 Jul 2024 22:54:45 +0300 Subject: [PATCH 23/36] str --- utils/compare_dependency_constraints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/compare_dependency_constraints.py b/utils/compare_dependency_constraints.py index a7a533391d306..8a841754523f4 100644 --- a/utils/compare_dependency_constraints.py +++ b/utils/compare_dependency_constraints.py @@ -50,7 +50,7 @@ class Discrepancy(NamedTuple): in_native: str | None = None def __str__(self) -> str: - return f"{self.dependency}: {self.in_image or 'missing'} in {self.image}, {self.in_native or 'missing'} in native" + return f"{self.dependency}: {self.in_image or 'missing'} in {self.image} image, {self.in_native or 'missing'} in Native image" def find_library_line_number(lib_name: str, file_path: Path): """ From 36a9adaac83cb347debd577ea61eee40cc3e29c8 Mon Sep 17 00:00:00 2001 From: Shmuel Kroizer <69422117+shmuel44@users.noreply.github.com> Date: Thu, 18 Jul 2024 14:10:50 +0300 Subject: [PATCH 24/36] Update .github/workflows/sync_native_dependency.yml Co-authored-by: dorschw <81086590+dorschw@users.noreply.github.com> --- .github/workflows/sync_native_dependency.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/sync_native_dependency.yml b/.github/workflows/sync_native_dependency.yml index d0737fbadfc5d..14b9ddf65c675 100644 --- a/.github/workflows/sync_native_dependency.yml +++ b/.github/workflows/sync_native_dependency.yml @@ -21,7 +21,6 @@ jobs: - name: Run compare_dependency_constraints Script. run: | ls - echo "==== $(date): Starting environment setup... ====" cd "$GITHUB_WORKSPACE" echo "==== $(date): Running chceking Native dependency ====" pipenv run python utils/compare_dependency_constraints.py \ No newline at end of file From 19fbf10ca847fb54ef912826e1c617ebb1091bd1 Mon Sep 17 00:00:00 2001 From: Shmuel Kroizer <69422117+shmuel44@users.noreply.github.com> Date: Thu, 18 Jul 2024 14:11:02 +0300 Subject: [PATCH 25/36] Update .github/workflows/sync_native_dependency.yml Co-authored-by: dorschw <81086590+dorschw@users.noreply.github.com> --- .github/workflows/sync_native_dependency.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/sync_native_dependency.yml b/.github/workflows/sync_native_dependency.yml index 14b9ddf65c675..6d29996e74ab0 100644 --- a/.github/workflows/sync_native_dependency.yml +++ b/.github/workflows/sync_native_dependency.yml @@ -22,5 +22,4 @@ jobs: run: | ls cd "$GITHUB_WORKSPACE" - echo "==== $(date): Running chceking Native dependency ====" pipenv run python utils/compare_dependency_constraints.py \ No newline at end of file From 8286c18eb0a400503405768244d4703e494380ba Mon Sep 17 00:00:00 2001 From: Shmuel Kroizer <69422117+shmuel44@users.noreply.github.com> Date: Thu, 18 Jul 2024 14:11:09 +0300 Subject: [PATCH 26/36] Update utils/compare_dependency_constraints.py Co-authored-by: dorschw <81086590+dorschw@users.noreply.github.com> --- utils/compare_dependency_constraints.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/compare_dependency_constraints.py b/utils/compare_dependency_constraints.py index 8a841754523f4..484f50c8b5bdd 100644 --- a/utils/compare_dependency_constraints.py +++ b/utils/compare_dependency_constraints.py @@ -52,7 +52,7 @@ class Discrepancy(NamedTuple): def __str__(self) -> str: return f"{self.dependency}: {self.in_image or 'missing'} in {self.image} image, {self.in_native or 'missing'} in Native image" -def find_library_line_number(lib_name: str, file_path: Path): +def find_library_line_number(lib_name: str, file_path: Path) -> int | None: """ Searches for a library in the pyproject.toml or Pipfile file and returns the line number where it is found. From 70f60399e4f05c439be2a435ac9e5e1267c04bb2 Mon Sep 17 00:00:00 2001 From: Shmuel Kroizer <69422117+shmuel44@users.noreply.github.com> Date: Thu, 18 Jul 2024 14:14:56 +0300 Subject: [PATCH 27/36] Update utils/compare_dependency_constraints.py Co-authored-by: dorschw <81086590+dorschw@users.noreply.github.com> --- utils/compare_dependency_constraints.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/utils/compare_dependency_constraints.py b/utils/compare_dependency_constraints.py index 484f50c8b5bdd..b91d6ffbabc1f 100644 --- a/utils/compare_dependency_constraints.py +++ b/utils/compare_dependency_constraints.py @@ -63,8 +63,7 @@ def find_library_line_number(lib_name: str, file_path: Path) -> int | None: Returns: - The line number containing the library name, or None if the library is not found. """ - with file_path.open('r') as file: - for line_number, line in enumerate(file, start=1): # Start counting from line 1 + for line_number, line in enumerate(file_path.read_text().splitlines(), start=1): # Start counting from line 1 if lib_name in line: return line_number From 622511efcb054248902557d11277337323068355 Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Thu, 18 Jul 2024 15:01:54 +0300 Subject: [PATCH 28/36] CR --- .github/workflows/sync_native_dependency.yml | 6 +- utils/compare_dependency_constraints.py | 77 ++++++++++++-------- 2 files changed, 48 insertions(+), 35 deletions(-) diff --git a/.github/workflows/sync_native_dependency.yml b/.github/workflows/sync_native_dependency.yml index 6d29996e74ab0..445167f4c29d9 100644 --- a/.github/workflows/sync_native_dependency.yml +++ b/.github/workflows/sync_native_dependency.yml @@ -9,10 +9,10 @@ jobs: if: github.repository == 'demisto/dockerFiles' steps: - uses: actions/checkout@v3 - - name: Set up Python 3.11 + - name: Set up Python uses: actions/setup-python@v3 with: - python-version: "3.11" + python-version: "3.*" - name: Install dependencies with pipenv run: | python -m pip install --upgrade pip @@ -20,6 +20,4 @@ jobs: pipenv install - name: Run compare_dependency_constraints Script. run: | - ls - cd "$GITHUB_WORKSPACE" pipenv run python utils/compare_dependency_constraints.py \ No newline at end of file diff --git a/utils/compare_dependency_constraints.py b/utils/compare_dependency_constraints.py index b91d6ffbabc1f..a701e0cea8dc8 100644 --- a/utils/compare_dependency_constraints.py +++ b/utils/compare_dependency_constraints.py @@ -7,28 +7,51 @@ DOCKER_FOLDER = Path(__file__).parent.parent / "docker" NATIVE_IMAGE = "py3-native" +PYPROJECT = "pyproject.toml" +PIPFILE = "Pipfile" -def parse_constraints(dir_name: str) -> dict[str, str]: +class Discrepancy(NamedTuple): + dependency: str + image: str + path: Path + in_image: str | None = None + in_native: str | None = None + + def __str__(self) -> str: + return ( + f"{self.dependency} is {self.in_image or 'missing'} in {self.image}, " + f"but {self.in_native or 'missing'} in the native image. " + "This discrepancy may cause issues when running content." + ) +def get_dependency_file_path(dir_name: str) -> Path: dir_path = DOCKER_FOLDER / dir_name + if not dir_path.exists(): raise FileNotFoundError(dir_path) - pip_path = dir_path / "Pipfile" - pyproject_path = dir_path / "pyproject.toml" + pip_path = dir_path / PIPFILE + pyproject_path = dir_path / PYPROJECT if pip_path.exists() and pyproject_path.exists(): raise ValueError( - f"Can't have both pyproject and Pipfile in a dockerfile folder ({dir_name})" + f"Can't have both pyproject and Pipfile in a dockerfile folder ({dir_path})" ) - if pip_path.exists(): - return lower_dict_keys(_parse_pipfile(pip_path)) + return pip_path if pyproject_path.exists(): - return lower_dict_keys(_parse_pyproject(pyproject_path)) + return pyproject_path + + raise ValueError(f"Neither pyproject nor Pipfile found in {dir_path}") + +def parse_constraints(name: str) -> dict[str, str]: + path = get_dependency_file_path(name) + if path.suffix == PIPFILE: + return lower_dict_keys(_parse_pipfile(path)) + + return lower_dict_keys(_parse_pyproject(path)) - raise ValueError(f"Neither pyproject nor Pipfile found in {dir_name}") def _parse_pipfile(path: Path) -> dict[str, str]: @@ -43,16 +66,7 @@ def lower_dict_keys(dictionary: dict[str, Any]) -> dict[str, Any]: return {k.lower(): v for k, v in dictionary.items()} -class Discrepancy(NamedTuple): - dependency: str - image: str - in_image: str | None = None - in_native: str | None = None - - def __str__(self) -> str: - return f"{self.dependency}: {self.in_image or 'missing'} in {self.image} image, {self.in_native or 'missing'} in Native image" - -def find_library_line_number(lib_name: str, file_path: Path) -> int | None: +def find_library_line_number(lib_name: str, file_path: Path) -> int: """ Searches for a library in the pyproject.toml or Pipfile file and returns the line number where it is found. @@ -61,13 +75,16 @@ def find_library_line_number(lib_name: str, file_path: Path) -> int | None: - file_path: The directory containing the pyproject.toml or Pipfile. Returns: - - The line number containing the library name, or None if the library is not found. + - The line number containing the library name, or 1 if the library is not found. """ - for line_number, line in enumerate(file_path.read_text().splitlines(), start=1): # Start counting from line 1 - if lib_name in line: - return line_number + for line_number, line in enumerate( + file_path.read_text().splitlines(), start=1 + ): # Start counting from line 1 + if lib_name in line: + return line_number + + return 1 # default - return None def compare_constraints(images_contained_in_native: list[str]): native_constraints = ( @@ -78,7 +95,7 @@ def compare_constraints(images_contained_in_native: list[str]): native_constraint_keys = set(native_constraints.keys()) discrepancies: list[Discrepancy] = [] for image in images_contained_in_native: - + path = get_dependency_file_path(image) constraints = parse_constraints(image) constraint_keys = set(constraints.keys()) @@ -88,6 +105,7 @@ def compare_constraints(images_contained_in_native: list[str]): dependency=dependency, image=image, in_image=constraints[dependency], + path=path ) for dependency in sorted( constraint_keys.difference(native_constraint_keys) @@ -101,6 +119,7 @@ def compare_constraints(images_contained_in_native: list[str]): image=image, in_image=constraints[dependency], in_native=native_constraints[dependency], + path=path ) for dependency in sorted( constraint_keys.intersection(native_constraint_keys) @@ -110,15 +129,11 @@ def compare_constraints(images_contained_in_native: list[str]): ) for discrepancy in discrepancies: - file_pyproject_path = DOCKER_FOLDER / f"{discrepancy.image}/pyproject.toml" - if file_pyproject_path.exists(): - file_path = file_pyproject_path - else: - file_path = DOCKER_FOLDER / f"{discrepancy.image}/Pipfile" - line_number = find_library_line_number(discrepancy.dependency, file_path) + + line_number = find_library_line_number(discrepancy.dependency, discrepancy.path) print( - f"::error file={file_path},line={line_number},endLine={line_number},title=Native Image Discrepancy::{discrepancy}" + f"::error file={discrepancy.path},line={line_number},endLine={line_number},title=Native Image Discrepancy::{discrepancy}" ) return int(bool(discrepancies)) From 5268bcd5e468ed0d734c316b98a02432126e32c1 Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Thu, 18 Jul 2024 15:02:41 +0300 Subject: [PATCH 29/36] CR --- .github/workflows/sync_native_dependency.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sync_native_dependency.yml b/.github/workflows/sync_native_dependency.yml index 445167f4c29d9..949fd9940f0d7 100644 --- a/.github/workflows/sync_native_dependency.yml +++ b/.github/workflows/sync_native_dependency.yml @@ -18,6 +18,5 @@ jobs: python -m pip install --upgrade pip pip install pipenv pipenv install - - name: Run compare_dependency_constraints Script. - run: | - pipenv run python utils/compare_dependency_constraints.py \ No newline at end of file + - name: Compare dependency constraints + run: pipenv run python utils/compare_dependency_constraints.py \ No newline at end of file From 3d87c1bd045c75c22081be0dd80097be670d5b75 Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Mon, 12 Aug 2024 16:28:14 +0300 Subject: [PATCH 30/36] compares the dependencies of `python3-ubi` against `py3-tools-ubi` --- utils/compare_dependency_constraints.py | 88 +++++++++++++++++++++---- 1 file changed, 75 insertions(+), 13 deletions(-) diff --git a/utils/compare_dependency_constraints.py b/utils/compare_dependency_constraints.py index a701e0cea8dc8..3f8a9e52add50 100644 --- a/utils/compare_dependency_constraints.py +++ b/utils/compare_dependency_constraints.py @@ -7,28 +7,35 @@ DOCKER_FOLDER = Path(__file__).parent.parent / "docker" NATIVE_IMAGE = "py3-native" -PYPROJECT = "pyproject.toml" +PY3_TOOLS_UBI_IMAGE = "py3-tools-ubi" +PYTHON3_UBI_IMAGE = "python3-ubi" +PYPROJECT = "pyproject.toml" PIPFILE = "Pipfile" class Discrepancy(NamedTuple): + """Represents a discrepancy between dependencies in different images.""" + dependency: str image: str path: Path in_image: str | None = None - in_native: str | None = None + in_reference: str | None = None def __str__(self) -> str: return ( f"{self.dependency} is {self.in_image or 'missing'} in {self.image}, " - f"but {self.in_native or 'missing'} in the native image. " + f"but {self.in_reference or 'missing'} in the reference image. " "This discrepancy may cause issues when running content." ) + + def get_dependency_file_path(dir_name: str) -> Path: + """Returns the path to the dependency file (Pipfile or pyproject.toml) in the given directory.""" dir_path = DOCKER_FOLDER / dir_name if not dir_path.exists(): - raise FileNotFoundError(dir_path) + raise FileNotFoundError(f"Directory {dir_path} does not exist.") pip_path = dir_path / PIPFILE pyproject_path = dir_path / PYPROJECT @@ -42,10 +49,12 @@ def get_dependency_file_path(dir_name: str) -> Path: if pyproject_path.exists(): return pyproject_path - + raise ValueError(f"Neither pyproject nor Pipfile found in {dir_path}") + def parse_constraints(name: str) -> dict[str, str]: + """Parses the dependency constraints from the given image name.""" path = get_dependency_file_path(name) if path.suffix == PIPFILE: return lower_dict_keys(_parse_pipfile(path)) @@ -53,16 +62,18 @@ def parse_constraints(name: str) -> dict[str, str]: return lower_dict_keys(_parse_pyproject(path)) - def _parse_pipfile(path: Path) -> dict[str, str]: + """Parses the Pipfile and returns the dependencies.""" return toml.load(path).get("packages", {}) def _parse_pyproject(path: Path) -> dict[str, str]: + """Parses the pyproject.toml file and returns the dependencies.""" return toml.load(path).get("tool", {}).get("poetry", {}).get("dependencies", {}) def lower_dict_keys(dictionary: dict[str, Any]) -> dict[str, Any]: + """Converts all keys in the dictionary to lowercase.""" return {k.lower(): v for k, v in dictionary.items()} @@ -87,13 +98,38 @@ def find_library_line_number(lib_name: str, file_path: Path) -> int: def compare_constraints(images_contained_in_native: list[str]): + """Compares the dependency constraints between different images and reports discrepancies. + + This function compares the dependencies of the following images: + - `python3-ubi` + - `py3-tools-ubi` + - `native` + + against the dependencies of the images listed in `images_contained_in_native`. + + Additionally, it compares the dependencies of `python3-ubi` against `py3-tools-ubi`. + + Args: + images_contained_in_native (list[str]): A list of image names to compare against the native image. + + Returns: + int: Returns 1 if there are discrepancies, 0 otherwise. + """ native_constraints = ( - parse_constraints("python3-ubi") - | parse_constraints("py3-tools-ubi") + parse_constraints(PYTHON3_UBI_IMAGE) + | parse_constraints(PY3_TOOLS_UBI_IMAGE) | parse_constraints(NATIVE_IMAGE) ) + python3_ubi_constraints = parse_constraints(PYTHON3_UBI_IMAGE) + py3_tools_ubi_constraints = parse_constraints(PY3_TOOLS_UBI_IMAGE) + native_constraint_keys = set(native_constraints.keys()) + python3_ubi_keys = set(python3_ubi_constraints.keys()) + py3_tools_ubi_keys = set(py3_tools_ubi_constraints.keys()) + discrepancies: list[Discrepancy] = [] + + # Compare each image with native for image in images_contained_in_native: path = get_dependency_file_path(image) constraints = parse_constraints(image) @@ -105,7 +141,7 @@ def compare_constraints(images_contained_in_native: list[str]): dependency=dependency, image=image, in_image=constraints[dependency], - path=path + path=path, ) for dependency in sorted( constraint_keys.difference(native_constraint_keys) @@ -118,8 +154,8 @@ def compare_constraints(images_contained_in_native: list[str]): dependency=dependency, image=image, in_image=constraints[dependency], - in_native=native_constraints[dependency], - path=path + in_reference=native_constraints[dependency], + path=path, ) for dependency in sorted( constraint_keys.intersection(native_constraint_keys) @@ -128,10 +164,35 @@ def compare_constraints(images_contained_in_native: list[str]): ) ) - for discrepancy in discrepancies: + # Compare py3-tools-ubi with python3-ubi + discrepancies.extend( # py3-tools-ubi dependencies missing from python3-ubi + ( + Discrepancy( + dependency=dependency, + image=PY3_TOOLS_UBI_IMAGE, + in_image=py3_tools_ubi_constraints[dependency], + in_reference=python3_ubi_constraints[dependency], + path=get_dependency_file_path(PY3_TOOLS_UBI_IMAGE), + ) + for dependency in sorted(py3_tools_ubi_keys.difference(python3_ubi_keys)) + ) + ) + discrepancies.extend( # shared dependencies with python3-ubi, different versions + ( + Discrepancy( + dependency=dependency, + image=PY3_TOOLS_UBI_IMAGE, + in_image=py3_tools_ubi_constraints[dependency], + in_reference=python3_ubi_constraints[dependency], + path=get_dependency_file_path(PY3_TOOLS_UBI_IMAGE), + ) + for dependency in sorted(py3_tools_ubi_keys.intersection(python3_ubi_keys)) + if py3_tools_ubi_constraints[dependency] != python3_ubi_constraints[dependency] + ) + ) + for discrepancy in discrepancies: line_number = find_library_line_number(discrepancy.dependency, discrepancy.path) - print( f"::error file={discrepancy.path},line={line_number},endLine={line_number},title=Native Image Discrepancy::{discrepancy}" ) @@ -139,6 +200,7 @@ def compare_constraints(images_contained_in_native: list[str]): def load_native_image_conf() -> list[str]: + """Returns the supported docker images by the native image from a remote JSON file.""" return json.loads( requests.get( "https://raw.githubusercontent.com/demisto/content/master/Tests/docker_native_image_config.json", From 1a0111c6f794e4374c3726aa24471285fd496dd0 Mon Sep 17 00:00:00 2001 From: Shmuel Kroizer <69422117+shmuel44@users.noreply.github.com> Date: Mon, 12 Aug 2024 16:36:32 +0300 Subject: [PATCH 31/36] Update docker/py3-native/pyproject.toml Co-authored-by: dorschw <81086590+dorschw@users.noreply.github.com> --- docker/py3-native/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/py3-native/pyproject.toml b/docker/py3-native/pyproject.toml index 9e0738ef9de00..faaca7d9fb018 100644 --- a/docker/py3-native/pyproject.toml +++ b/docker/py3-native/pyproject.toml @@ -45,7 +45,7 @@ parse-emails = "*" slack-sdk = {extras = ["optional"], version = "*"} pyjwt = {extras = ["crypto"],version = "*"} google-api-python-client = "*" -exchangelib = ">=5.4.0" +exchangelib = ">=5.4.1" requests-oauthlib = "*" oauthlib = {extras = ["signedtoken"], version = "*"} requests-ntlm = "==1.1.0" From 79e9bb86a8e6d55846934ac1db49b328f07f884e Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Mon, 12 Aug 2024 16:41:24 +0300 Subject: [PATCH 32/36] .get(dependency) --- utils/compare_dependency_constraints.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/utils/compare_dependency_constraints.py b/utils/compare_dependency_constraints.py index 3f8a9e52add50..a20c531d6a75f 100644 --- a/utils/compare_dependency_constraints.py +++ b/utils/compare_dependency_constraints.py @@ -25,7 +25,7 @@ class Discrepancy(NamedTuple): def __str__(self) -> str: return ( f"{self.dependency} is {self.in_image or 'missing'} in {self.image}, " - f"but {self.in_reference or 'missing'} in the reference image. " + f"but {self.in_reference or 'missing'} in the {self.image} image. " "This discrepancy may cause issues when running content." ) @@ -170,8 +170,8 @@ def compare_constraints(images_contained_in_native: list[str]): Discrepancy( dependency=dependency, image=PY3_TOOLS_UBI_IMAGE, - in_image=py3_tools_ubi_constraints[dependency], - in_reference=python3_ubi_constraints[dependency], + in_image=py3_tools_ubi_constraints.get(dependency), + in_reference=python3_ubi_constraints.get(dependency), path=get_dependency_file_path(PY3_TOOLS_UBI_IMAGE), ) for dependency in sorted(py3_tools_ubi_keys.difference(python3_ubi_keys)) @@ -182,12 +182,12 @@ def compare_constraints(images_contained_in_native: list[str]): Discrepancy( dependency=dependency, image=PY3_TOOLS_UBI_IMAGE, - in_image=py3_tools_ubi_constraints[dependency], - in_reference=python3_ubi_constraints[dependency], + in_image=py3_tools_ubi_constraints.get(dependency), + in_reference=python3_ubi_constraints.get(dependency), path=get_dependency_file_path(PY3_TOOLS_UBI_IMAGE), ) for dependency in sorted(py3_tools_ubi_keys.intersection(python3_ubi_keys)) - if py3_tools_ubi_constraints[dependency] != python3_ubi_constraints[dependency] + if py3_tools_ubi_constraints.get(dependency) != python3_ubi_constraints.get(dependency) ) ) From 30267daa7abc169af4d9c5fd6b53d1139c963dbd Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Mon, 12 Aug 2024 16:45:19 +0300 Subject: [PATCH 33/36] python3-ubi > py3-tools --- utils/compare_dependency_constraints.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/utils/compare_dependency_constraints.py b/utils/compare_dependency_constraints.py index a20c531d6a75f..e4822982185f0 100644 --- a/utils/compare_dependency_constraints.py +++ b/utils/compare_dependency_constraints.py @@ -8,7 +8,7 @@ DOCKER_FOLDER = Path(__file__).parent.parent / "docker" NATIVE_IMAGE = "py3-native" PY3_TOOLS_UBI_IMAGE = "py3-tools-ubi" -PYTHON3_UBI_IMAGE = "python3-ubi" +PYTHON3_UBI_IMAGE = "py3-tools" PYPROJECT = "pyproject.toml" PIPFILE = "Pipfile" @@ -101,13 +101,13 @@ def compare_constraints(images_contained_in_native: list[str]): """Compares the dependency constraints between different images and reports discrepancies. This function compares the dependencies of the following images: - - `python3-ubi` + - `py3-tools` - `py3-tools-ubi` - `native` against the dependencies of the images listed in `images_contained_in_native`. - Additionally, it compares the dependencies of `python3-ubi` against `py3-tools-ubi`. + Additionally, it compares the dependencies of `py3-tools` against `py3-tools-ubi`. Args: images_contained_in_native (list[str]): A list of image names to compare against the native image. @@ -164,8 +164,8 @@ def compare_constraints(images_contained_in_native: list[str]): ) ) - # Compare py3-tools-ubi with python3-ubi - discrepancies.extend( # py3-tools-ubi dependencies missing from python3-ubi + # Compare py3-tools-ubi with py3-tools + discrepancies.extend( # py3-tools-ubi dependencies missing from py3-tools ( Discrepancy( dependency=dependency, @@ -177,7 +177,7 @@ def compare_constraints(images_contained_in_native: list[str]): for dependency in sorted(py3_tools_ubi_keys.difference(python3_ubi_keys)) ) ) - discrepancies.extend( # shared dependencies with python3-ubi, different versions + discrepancies.extend( # shared dependencies with py3-tools, different versions ( Discrepancy( dependency=dependency, From 68467b901fc850bd2ed84d91747bbf1f99eb4a60 Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Mon, 12 Aug 2024 16:54:01 +0300 Subject: [PATCH 34/36] reference_image=PY3_TOOLS_IMAGE, --- utils/compare_dependency_constraints.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/utils/compare_dependency_constraints.py b/utils/compare_dependency_constraints.py index e4822982185f0..723ae2360b195 100644 --- a/utils/compare_dependency_constraints.py +++ b/utils/compare_dependency_constraints.py @@ -8,7 +8,7 @@ DOCKER_FOLDER = Path(__file__).parent.parent / "docker" NATIVE_IMAGE = "py3-native" PY3_TOOLS_UBI_IMAGE = "py3-tools-ubi" -PYTHON3_UBI_IMAGE = "py3-tools" +PY3_TOOLS_IMAGE = "py3-tools" PYPROJECT = "pyproject.toml" PIPFILE = "Pipfile" @@ -18,6 +18,7 @@ class Discrepancy(NamedTuple): dependency: str image: str + reference_image: str path: Path in_image: str | None = None in_reference: str | None = None @@ -25,7 +26,7 @@ class Discrepancy(NamedTuple): def __str__(self) -> str: return ( f"{self.dependency} is {self.in_image or 'missing'} in {self.image}, " - f"but {self.in_reference or 'missing'} in the {self.image} image. " + f"but {self.in_reference or 'missing'} in the {self.reference_image} image. " "This discrepancy may cause issues when running content." ) @@ -116,15 +117,15 @@ def compare_constraints(images_contained_in_native: list[str]): int: Returns 1 if there are discrepancies, 0 otherwise. """ native_constraints = ( - parse_constraints(PYTHON3_UBI_IMAGE) + parse_constraints(PY3_TOOLS_IMAGE) | parse_constraints(PY3_TOOLS_UBI_IMAGE) | parse_constraints(NATIVE_IMAGE) ) - python3_ubi_constraints = parse_constraints(PYTHON3_UBI_IMAGE) + py3_tools_constraints = parse_constraints(PY3_TOOLS_IMAGE) py3_tools_ubi_constraints = parse_constraints(PY3_TOOLS_UBI_IMAGE) native_constraint_keys = set(native_constraints.keys()) - python3_ubi_keys = set(python3_ubi_constraints.keys()) + py3_tools_keys = set(py3_tools_constraints.keys()) py3_tools_ubi_keys = set(py3_tools_ubi_constraints.keys()) discrepancies: list[Discrepancy] = [] @@ -140,6 +141,7 @@ def compare_constraints(images_contained_in_native: list[str]): Discrepancy( dependency=dependency, image=image, + reference_image="native", in_image=constraints[dependency], path=path, ) @@ -153,6 +155,7 @@ def compare_constraints(images_contained_in_native: list[str]): Discrepancy( dependency=dependency, image=image, + reference_image="native", in_image=constraints[dependency], in_reference=native_constraints[dependency], path=path, @@ -170,11 +173,12 @@ def compare_constraints(images_contained_in_native: list[str]): Discrepancy( dependency=dependency, image=PY3_TOOLS_UBI_IMAGE, + reference_image=PY3_TOOLS_IMAGE, in_image=py3_tools_ubi_constraints.get(dependency), - in_reference=python3_ubi_constraints.get(dependency), + in_reference=py3_tools_constraints.get(dependency), path=get_dependency_file_path(PY3_TOOLS_UBI_IMAGE), ) - for dependency in sorted(py3_tools_ubi_keys.difference(python3_ubi_keys)) + for dependency in sorted(py3_tools_ubi_keys.difference(py3_tools_keys)) ) ) discrepancies.extend( # shared dependencies with py3-tools, different versions @@ -182,12 +186,13 @@ def compare_constraints(images_contained_in_native: list[str]): Discrepancy( dependency=dependency, image=PY3_TOOLS_UBI_IMAGE, + reference_image=PY3_TOOLS_IMAGE, in_image=py3_tools_ubi_constraints.get(dependency), - in_reference=python3_ubi_constraints.get(dependency), + in_reference=py3_tools_constraints.get(dependency), path=get_dependency_file_path(PY3_TOOLS_UBI_IMAGE), ) - for dependency in sorted(py3_tools_ubi_keys.intersection(python3_ubi_keys)) - if py3_tools_ubi_constraints.get(dependency) != python3_ubi_constraints.get(dependency) + for dependency in sorted(py3_tools_ubi_keys.intersection(py3_tools_keys)) + if py3_tools_ubi_constraints.get(dependency) != py3_tools_constraints.get(dependency) ) ) From 6b47d36e6895fab4f57b7c39bd7eb0f2c3ed7a8e Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Mon, 12 Aug 2024 17:22:40 +0300 Subject: [PATCH 35/36] NATIVE_IMAGE --- utils/compare_dependency_constraints.py | 104 ++++++++++++++---------- 1 file changed, 61 insertions(+), 43 deletions(-) diff --git a/utils/compare_dependency_constraints.py b/utils/compare_dependency_constraints.py index 723ae2360b195..65e525fd51892 100644 --- a/utils/compare_dependency_constraints.py +++ b/utils/compare_dependency_constraints.py @@ -98,7 +98,7 @@ def find_library_line_number(lib_name: str, file_path: Path) -> int: return 1 # default -def compare_constraints(images_contained_in_native: list[str]): +def compare_constraints(images_contained_in_native: list[str]) -> int: """Compares the dependency constraints between different images and reports discrepancies. This function compares the dependencies of the following images: @@ -116,6 +116,7 @@ def compare_constraints(images_contained_in_native: list[str]): Returns: int: Returns 1 if there are discrepancies, 0 otherwise. """ + native_constraints = ( parse_constraints(PY3_TOOLS_IMAGE) | parse_constraints(PY3_TOOLS_UBI_IMAGE) @@ -123,51 +124,72 @@ def compare_constraints(images_contained_in_native: list[str]): ) py3_tools_constraints = parse_constraints(PY3_TOOLS_IMAGE) py3_tools_ubi_constraints = parse_constraints(PY3_TOOLS_UBI_IMAGE) + discrepancies: list[Discrepancy] = [] + + for image in images_contained_in_native: + discrepancies.extend(compare_with_native(image, native_constraints)) + + discrepancies.extend( + compare_py3_tools_with_ubi(py3_tools_constraints, py3_tools_ubi_constraints) + ) + + + for discrepancy in discrepancies: + line_number = find_library_line_number(discrepancy.dependency, discrepancy.path) + print( + f"::error file={discrepancy.path},line={line_number},endLine={line_number},title=Native Image Discrepancy::{discrepancy}" + ) + return int(bool(discrepancies)) + +def compare_with_native(image: str, native_constraints: dict) -> list[Discrepancy]: + path = get_dependency_file_path(image) + constraints = parse_constraints(image) + constraint_keys = set(constraints.keys()) native_constraint_keys = set(native_constraints.keys()) - py3_tools_keys = set(py3_tools_constraints.keys()) - py3_tools_ubi_keys = set(py3_tools_ubi_constraints.keys()) discrepancies: list[Discrepancy] = [] - # Compare each image with native - for image in images_contained_in_native: - path = get_dependency_file_path(image) - constraints = parse_constraints(image) - constraint_keys = set(constraints.keys()) - - discrepancies.extend( # image dependencies missing from native - ( - Discrepancy( - dependency=dependency, - image=image, - reference_image="native", - in_image=constraints[dependency], - path=path, - ) - for dependency in sorted( - constraint_keys.difference(native_constraint_keys) - ) + discrepancies.extend( # image dependencies missing from native + ( + Discrepancy( + dependency=dependency, + image=image, + reference_image=NATIVE_IMAGE, + in_image=constraints[dependency], + path=path, ) + for dependency in sorted(constraint_keys.difference(native_constraint_keys)) ) - discrepancies.extend( # shared dependencies with native, different versions - ( - Discrepancy( - dependency=dependency, - image=image, - reference_image="native", - in_image=constraints[dependency], - in_reference=native_constraints[dependency], - path=path, - ) - for dependency in sorted( - constraint_keys.intersection(native_constraint_keys) - ) - if constraints[dependency] != native_constraints[dependency] + ) + discrepancies.extend( # shared dependencies with native, different versions + ( + Discrepancy( + dependency=dependency, + image=image, + reference_image=NATIVE_IMAGE, + in_image=constraints[dependency], + in_reference=native_constraints[dependency], + path=path, ) + for dependency in sorted( + constraint_keys.intersection(native_constraint_keys) + ) + if constraints[dependency] != native_constraints[dependency] ) + ) + + return discrepancies + + +def compare_py3_tools_with_ubi( + py3_tools_constraints: dict, py3_tools_ubi_constraints: dict +) -> list[Discrepancy]: + py3_tools_keys = set(py3_tools_constraints.keys()) + py3_tools_ubi_keys = set(py3_tools_ubi_constraints.keys()) + + discrepancies: list[Discrepancy] = [] - # Compare py3-tools-ubi with py3-tools discrepancies.extend( # py3-tools-ubi dependencies missing from py3-tools ( Discrepancy( @@ -192,16 +214,12 @@ def compare_constraints(images_contained_in_native: list[str]): path=get_dependency_file_path(PY3_TOOLS_UBI_IMAGE), ) for dependency in sorted(py3_tools_ubi_keys.intersection(py3_tools_keys)) - if py3_tools_ubi_constraints.get(dependency) != py3_tools_constraints.get(dependency) + if py3_tools_ubi_constraints.get(dependency) + != py3_tools_constraints.get(dependency) ) ) - for discrepancy in discrepancies: - line_number = find_library_line_number(discrepancy.dependency, discrepancy.path) - print( - f"::error file={discrepancy.path},line={line_number},endLine={line_number},title=Native Image Discrepancy::{discrepancy}" - ) - return int(bool(discrepancies)) + return discrepancies def load_native_image_conf() -> list[str]: From 9e48b25b68717429eefcbc213c3d70623330900d Mon Sep 17 00:00:00 2001 From: shmuel44 Date: Mon, 12 Aug 2024 17:26:07 +0300 Subject: [PATCH 36/36] ruff --- utils/compare_dependency_constraints.py | 1 - 1 file changed, 1 deletion(-) diff --git a/utils/compare_dependency_constraints.py b/utils/compare_dependency_constraints.py index 65e525fd51892..b4b6afc9d453c 100644 --- a/utils/compare_dependency_constraints.py +++ b/utils/compare_dependency_constraints.py @@ -133,7 +133,6 @@ def compare_constraints(images_contained_in_native: list[str]) -> int: compare_py3_tools_with_ubi(py3_tools_constraints, py3_tools_ubi_constraints) ) - for discrepancy in discrepancies: line_number = find_library_line_number(discrepancy.dependency, discrepancy.path) print(