update-dockerfile-update-workflow

.github/workflows/create-update-dockerfiles-PR.yml

@@ -2,8 +2,9 @@ name: Create update dockerfiles PR
 
 on:
   workflow_run:
-    workflows: [Update external base images, Update internal base images]
-    types: completed
+    workflows: [Update external base images test, Update internal base images test]
+    types:
+      - completed
   workflow_dispatch:
 
 jobs:
@@ -22,7 +23,7 @@ jobs:
           GITHUB_TOKEN: ${{secrets.CONTENT_BOT_TOKEN}}
         run: |
           git fetch
-          branches=$(git branch -r | grep autoupdate/)
+          branches=$(git branch -r | grep TESTautoTESTupdate/)
           matrix="[ "
           for branch in $(echo $branches); do
             exists=`gh pr list --state open -H ${branch#origin/}`
@@ -55,13 +56,13 @@ jobs:
         destination_branch: ""                                                    # If blank, default: master
         pr_title: "build(dockerfiles) update Dockerfile - ${{ matrix.branches }}" # Title of pull request
         pr_body: "This is automated PR to update dockerfiles base images\n${{ matrix.branches }}"
-    - name: approve and merge
-      env:
-        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        PR_URL: ${{ steps.open-pr.outputs.pr_url }}
-      if: ${{ steps.open-pr.outputs.pr_url }}
-      run: |
-        echo "Approving and merging"
-        gh pr review --approve "$PR_URL"
-        gh pr merge --auto --squash "$PR_URL" 
+    # - name: approve and merge
+    #   env:
+    #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    #     PR_URL: ${{ steps.open-pr.outputs.pr_url }}
+    #   if: ${{ steps.open-pr.outputs.pr_url }}
+    #   run: |
+    #     echo "Approving and merging"
+    #     gh pr review --approve "$PR_URL"
+    #     gh pr merge --auto --squash "$PR_URL" 
 

.github/workflows/update-external-base-images.yml

@@ -1,10 +1,13 @@
-name: Update external base images
+name: Update external base images test
 
 on:
-  schedule:
-    - cron: '0 0 * * *'
+  push:
+    branches:
+      - update-dockerfile-update-workflow
+  # schedule:
+  #   - cron: '0 0 * * *'
   workflow_dispatch:
-  
+
 
 jobs:
   update-docker-files:
@@ -24,7 +27,8 @@ jobs:
             git config --global user.name "auto dockerfiles update"
             echo "==== $(date): Starting pipenv setup... ===="
             python -m pip install --upgrade pip
-            pip install pipenv
+            pip install pipenv==2023.3.18
+            pip install poetry
             pipenv install
             echo "==== Finished ===="
             [[ ${{ vars.DISABLE_TIMESTAMP_AUTOUPDATES }} = 'true' ]] && tu_flag="--no-timestamp-updates"

.github/workflows/update-internal-base-images.yml

@@ -1,8 +1,12 @@
-name: Update internal base images
+name: Update internal base images test
 
+# on:
+#   schedule:
+#     - cron: '30 0,4,8,12,16,20 * * *'
 on:
-  schedule:
-    - cron: '30 0,4,8,12,16,20 * * *'
+  push:
+    branches:
+      - update-dockerfile-update-workflow
   workflow_dispatch:
 
 jobs:
@@ -22,7 +26,8 @@ jobs:
             git config --global user.name "auto dockerfiles update"
             echo "==== $(date): Starting pipenv setup... ===="
             python -m pip install --upgrade pip
-            pip install pipenv
+            pip install pipenv==2023.3.18
+            pip install poetry
             pipenv install
             echo "==== Finished ===="
             pipenv run python ./utils/auto_dockerfile_update/update_dockerfiles.py -t internal

utils/auto_dockerfile_update/get_dockerfiles.py

@@ -3,16 +3,17 @@
 import re
 from datetime import datetime
 from pathlib import Path
-from typing import List, Dict
+from typing import List, Dict, Tuple
 from configparser import ConfigParser, MissingSectionHeaderError
 from dateutil.parser import parse
 
-
+EXTRACT_PYTHON_VERSION_REGEX = re.compile(r"\d{1,3}.\d{1,3}.\d{1,3}")
 BASE_IMAGE_REGEX = re.compile(r"(?:FROM [\S]+)")
 INTERNAL_BASE_IMAGES = re.compile(r"(demisto\/|devdemisto\/)")
 LAST_MODIFIED_REGEX = re.compile(r"# Last modified: [^\n]*")
 
 
+
 def get_last_modified(docker_file_content: str) -> str:
     """
     Get the last modified section in dockerfile, if not exists return 1.1.2000 12:00:00 +00:00
@@ -90,7 +91,32 @@ def filter_ignored_files(files_list):
         print(f'could not read ignored config {str(e)}')
         return files_list
 
-
+def get_file_path_and_docker_version_if_exist(dockerfile: dict, latest_tag:str)-> Tuple:
+        """Gets the Pipfile or the pyproject.toml file path 
+           and the docker version from the Dockerfile.
+        Args:
+            dockerfile (str): A dict that represents the docker file.
+            latest_tag (str): latest tag string.
+        Returns:
+            Tuple[str,str,bool]: The file path, The docker file tag, if exists.
+        """
+        base_path = dockerfile["path"]
+        base_path=base_path.replace("/Dockerfile", "")
+        pipfile_path = glob(f"{base_path}/Pipfile", recursive=True)
+        pyproject_path = glob(f"{base_path}/pyproject.toml", recursive=True)
+        extracted_tag_search = re.search(EXTRACT_PYTHON_VERSION_REGEX, latest_tag)
+        if extracted_tag_search:
+            extracted_tag = extracted_tag_search.group(0)
+            if pipfile_path:
+                return pipfile_path[0],extracted_tag,True
+            elif pyproject_path:
+                return pyproject_path[0],extracted_tag,True
+            else:
+                print(f"[ERROR] Can't find Pipfile/pyproject file for {dockerfile['name']}.")
+        else:
+            print(f"[ERROR] Can't find docker tag for {dockerfile['name']}.")
+        return base_path,extracted_tag,False
+
 def get_docker_files(base_path="docker/", devonly=False, external=False, internal=False) -> List[Dict]:
     """
     Get all the relevant dockerfiles from the repository.

utils/auto_dockerfile_update/update_dockerfiles.py

@@ -1,19 +1,19 @@
 import os
-
 import argparse
-
-from get_dockerfiles import get_docker_files
+from get_dockerfiles import get_docker_files,get_file_path_and_docker_version_if_exist
 from get_latest_tag import get_latest_tag, parse_versions
-from typing import Dict, List
+from typing import Dict, List,Tuple
 import dateutil
 from git import Repo, GitCommandError
 import re
 from get_dockerfiles import LAST_MODIFIED_REGEX
 from datetime import datetime, timezone
 from functools import reduce
+import subprocess
 
 BATCH_SIZE = 1
-
+PYTHON_VERSION_PIPFILE_REGEX = re.compile(r"python_version = \"([^\"]+)\"")
+PYTHON_VERSION_PYPROJECT_REGEX = re.compile(r"python = \"([^\"]+)\"")
 
 def is_docker_file_outdated(dockerfile: Dict, latest_tag: str, last_updated: str = "", no_timestamp_updates=True) -> bool:
     """
@@ -40,13 +40,130 @@ def is_docker_file_outdated(dockerfile: Dict, latest_tag: str, last_updated: str
 
     return False
 
+def extract_current_python_version(file_path: str)->Tuple[List,str,bool]:
+    """Extract the current python version from Pipfile or the pyproject.toml.
+
+        Args:
+            file_path (str): The file path to the Pipfile or the pyproject.toml file.
+
+        Returns:
+            Tuple[List,str, bool]: The python version in list and str.
+    """
+    python_version = ""
+    try:
+        with open(file_path, "r") as f:
+            file_content = f.read()
+        if "Pipfile" in file_path:
+            python_version = re.search(PYTHON_VERSION_PIPFILE_REGEX, file_content)
+        elif "pyproject.toml" in file_path:
+            python_version = re.search(PYTHON_VERSION_PYPROJECT_REGEX, file_content)   
+    except Exception as e:
+        print(f"{e}: Can't read file {file_path}")
+    if python_version:
+        full_str_python_version = python_version.group(0)
+        string_version_number=python_version.group(1)
+        return string_version_number,full_str_python_version,False
+    else:
+        print(f"can't extract python version form:{file_path}")
+        return "","",False
+
+def replace_python_version(file_path: str, version: List,
+                           full_str_python_version:str)->bool:
+    """Replace the current python version in the Pipfile or the pyproject.toml.
+
+        Args:
+            file_path (str): The file path to the Pipfile or the pyproject.toml file.
+            version (List): The updated version.
+            full_str_python_version (str): The older version.
+    """
+    to_replace = False
+    with open(file_path, "r") as f:
+        file_content = f.read()
+        if "Pipfile" in file_path:
+            if full_str_python_version != f"python_version = \"{version[0]}.{version[1]}\"":
+                file_content=file_content.replace(full_str_python_version,f"python_version = \"{version[0]}.{version[1]}\"")
+                to_replace = True
+        elif "pyproject.toml" in file_path:
+            if full_str_python_version != f"python = \"~{version[0]}.{version[1]}\"":
+                file_content=file_content.replace(full_str_python_version,f"python = \"~{version[0]}.{version[1]}\"")
+                to_replace= True
+        if to_replace:
+            with open(file_path, "w") as f:
+                f.write(file_content)
+            return True
+        return False
+
+def change_pyproject_or_pipfile(file_path, str_version) -> Tuple[bool,str]:
+    """Replace the current python version in the Pipfile or the pyproject.toml.
+
+        Args:
+            file_path (str): The file path to the Pipfile or the pyproject.toml file.
+            version (List): The updated version.
+            full_str_python_version (str): The older version.
+    """
+    version = str_version.split(".")
+    current_version, full_str_python_version, success_extraction=extract_current_python_version(file_path)
+    if success_extraction:
+        result=replace_python_version(file_path, version, full_str_python_version)
+        return result,current_version
+    return False, []
+
+def run_lock(base_path_docker:str,pipfile_or_pyproject_path:str)->bool:
+    """Runs poetry lock --no-update or pipfile lock --keep-outdated.
+
+        Args:
+            base_path_docker (str): The DockerFile path.
+            pipfile_or_pyproject_path (str): The file path to the Pipfile or the pyproject.toml file.
+    """
+    base_path=base_path_docker.replace("/Dockerfile", "")
+    current_directory = os.getcwd()
+    os.chdir(f"{current_directory}/" + base_path)
+    try:
+        if "Pipfile" in pipfile_or_pyproject_path:
+            # waits for the process to end.
+            completed_process = subprocess.run(["pipenv", "lock", "--keep-outdated"], check=True)
+            if completed_process.returncode != 0 :
+                return False
+        elif "pyproject.toml" in pipfile_or_pyproject_path:
+            completed_process = subprocess.run(["poetry", "lock", "--no-update"], check=True)
+            if completed_process.returncode != 0 :
+                return False
+    except subprocess.CalledProcessError as e:
+        print(f"[ERROR] Lock failed with error: {e}")
+    except TimeoutError as e:
+        print(f"[ERROR] Got time out error: {e} for {base_path_docker}")
+    except Exception as e:
+        print(f"[ERROR] {e}: for {base_path_docker}")
+    os.chdir(current_directory)
+    return True
+
+def update_python_version_pyproject_or_pipfile(dockerfile: Dict, latest_tag: str)-> None:
+    """
+    Updating dockerfile content with the latest tag
+    Args:
+        dockerfile (Dict): Dockerfile dict.
+        latest_tag (str): latest tag string.
+    Returns:
+        None
+    """
+    update_result = False
+    image_name = dockerfile['image_name']
+    if "python" in image_name:
+        path,tag, is_exist=get_file_path_and_docker_version_if_exist(dockerfile,latest_tag)
+        if is_exist:
+            update_result, old_version = change_pyproject_or_pipfile(path,tag)
+        if update_result:
+            lock_result=run_lock(dockerfile['path'],path)
+            if not lock_result:
+                print(f"[ERROR] Got Error with lock for: {image_name} revert pipfile/pyproject.toml changes")
+                change_pyproject_or_pipfile(path,old_version)
 
 def update_dockerfile(dockerfile: Dict, latest_tag: str) -> None:
     """
     Updating dockerfile content with the latest tag
     Args:
-        dockerfile (Dict): Dockerfile dict
-        latest_tag (str): latest tag string
+        dockerfile (Dict): Dockerfile dict.
+        latest_tag (str): latest tag string.
     Returns:
         None
     """
@@ -86,9 +203,10 @@ def update_external_base_dockerfiles(git_repo: Repo, no_timestamp_updates=True)
         latest_tag_last_updated = latest_tag.get('last_updated', '')
 
         if is_docker_file_outdated(file, latest_tag_name, latest_tag_last_updated, no_timestamp_updates):
-            branch_name = fr"autoupdate/Update_{file['repo']}_{file['image_name']}_from_{file['tag']}_to_{latest_tag_name}"
+            branch_name = fr"TESTautoTESTupdate/Update_{file['repo']}_{file['image_name']}_from_{file['tag']}_to_{latest_tag_name}"
             update_and_push_dockerfiles(git_repo, branch_name, [file], latest_tag_name)
             print(f"Updated {file['path']}")
+    print("Finished to update dockerfiles")
 
 
 def create_dependencies_json(all_docker_files: List[Dict]) -> Dict:
@@ -133,8 +251,9 @@ def update_internal_base_dockerfile(git_repo: Repo) -> None:
                           is_docker_file_outdated(file, latest_tag_name, latest_tag_last_updated)]
         for batch_slice in batch(outdated_files, BATCH_SIZE):
             image_names = reduce(lambda a, b: f"{a}-{b}", [file['name'] for file in batch_slice])
-            branch_name = fr"autoupdate/{base_image}_{image_names}_{latest_tag_name}"
+            branch_name = fr"TESTautoTESTupdate/{base_image}_{image_names}_{latest_tag_name}"
             update_and_push_dockerfiles(git_repo, branch_name, batch_slice, latest_tag_name)
+        print("Finished to update dockerfiles")
 
 
 def update_and_push_dockerfiles(git_repo: Repo, branch_name: str, files: List[Dict], latest_tag_name: str) -> None:
@@ -152,15 +271,17 @@ def update_and_push_dockerfiles(git_repo: Repo, branch_name: str, files: List[Di
     print(f"Trying to create new branch: {branch_name}")
     original_branch = git_repo.active_branch
     if branch_name in git_repo.git.branch("--all"):
-        print("Branch already exits.")
+        print("Branch already exists.")
         return
     try:
         branch = git_repo.create_head(branch_name)
         branch.checkout()
 
         for file in files:
+            update_python_version_pyproject_or_pipfile(file, latest_tag_name)
             update_dockerfile(file, latest_tag_name)
 
+        print(git_repo.index.diff(git_repo.head.commit))
         git_repo.git.add("*")
         git_repo.git.commit(m=f"Update Dockerfiles")
         git_repo.git.push('--set-upstream', 'origin', branch)