[AUTO] Updated input manifests. (#2596)

* Updated manifests. Signed-off-by: opensearch-ci-bot <[email protected]> * Remove 2.2.2 from the check for build as 2.3.0 is about to release Signed-off-by: Peter Zhu <[email protected]> * Add common-utils Signed-off-by: Peter Zhu <[email protected]> Signed-off-by: opensearch-ci-bot <[email protected]> Signed-off-by: Peter Zhu <[email protected]> Co-authored-by: opensearch-ci-bot <[email protected]> Co-authored-by: Peter Zhu <[email protected]>
opensearch-project · Sep 9, 2022 · 4504360 · 4504360
1 parent 67e71fe
commit 4504360
Show file tree

Hide file tree

Showing 11 changed files with 242 additions and 75 deletions.
diff --git a/.github/workflows/increment-plugin-versions.yml b/.github/workflows/increment-plugin-versions.yml
@@ -39,9 +39,8 @@ jobs:
           - {repo: security}
           - {repo: sql}
         branch:
-          - '2.x'
-          - '2.2'
           - '1.3'
+          - 2.x
         exclude:
           - {entry: {repo: geospatial}, branch: '1.3'}
           - {entry: {repo: notifications, path: notifications}, branch: '1.3'}

diff --git a/Pipfile b/Pipfile
@@ -28,6 +28,7 @@ pytablewriter = "*"
 typed-ast = "*"
 zipp = "*"
 importlib-metadata = "*"
+ruamel-yaml = "*"
 
 [dev-packages]
 

diff --git a/Pipfile.lock b/Pipfile.lock
diff --git a/jenkins/check-for-build.jenkinsfile b/jenkins/check-for-build.jenkinsfile
@@ -11,6 +11,8 @@ pipeline {
     }
     triggers {
         parameterizedCron '''
+            H 1 * * * %INPUT_MANIFEST=2.4.0/opensearch-2.4.0.yml;TARGET_JOB_NAME=distribution-build-opensearch
+            H 1 * * * %INPUT_MANIFEST=1.3.6/opensearch-1.3.6.yml;TARGET_JOB_NAME=distribution-build-opensearch
             H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch
             H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-dashboards-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards
         '''

diff --git a/manifests/1.3.6/opensearch-1.3.6.yml b/manifests/1.3.6/opensearch-1.3.6.yml
@@ -0,0 +1,16 @@
+---
+schema-version: '1.0'
+build:
+  name: OpenSearch
+  version: 1.3.6
+ci:
+  image:
+    name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2
+    args: -e JAVA_HOME=/opt/java/openjdk-11
+components:
+  - name: OpenSearch
+    repository: https://github.com/opensearch-project/OpenSearch.git
+    ref: '1.3'
+    checks:
+      - gradle:publish
+      - gradle:properties:version
diff --git a/manifests/2.2.2/opensearch-2.2.2.yml b/manifests/2.2.2/opensearch-2.2.2.yml
@@ -0,0 +1,16 @@
+---
+schema-version: '1.0'
+build:
+  name: OpenSearch
+  version: 2.2.2
+ci:
+  image:
+    name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2
+    args: -e JAVA_HOME=/opt/java/openjdk-17
+components:
+  - name: OpenSearch
+    repository: https://github.com/opensearch-project/OpenSearch.git
+    ref: '2.2'
+    checks:
+      - gradle:publish
+      - gradle:properties:version
diff --git a/manifests/2.4.0/opensearch-2.4.0.yml b/manifests/2.4.0/opensearch-2.4.0.yml
@@ -0,0 +1,24 @@
+---
+schema-version: '1.0'
+build:
+  name: OpenSearch
+  version: 2.4.0
+ci:
+  image:
+    name: opensearchstaging/ci-runner:ci-runner-centos7-opensearch-build-v2
+    args: -e JAVA_HOME=/opt/java/openjdk-17
+components:
+  - name: OpenSearch
+    repository: https://github.com/opensearch-project/OpenSearch.git
+    ref: 2.x
+    checks:
+      - gradle:publish
+      - gradle:properties:version
+  - name: common-utils
+    repository: https://github.com/opensearch-project/common-utils.git
+    ref: 2.x
+    platforms:
+      - linux
+    checks:
+      - gradle:publish
+      - gradle:properties:version
diff --git a/src/manifests_workflow/input_manifests.py b/src/manifests_workflow/input_manifests.py
@@ -11,6 +11,8 @@
 from abc import abstractmethod
 from typing import Dict, List, Type, Union
 
+import ruamel.yaml
+
 from manifests.input_manifest import InputComponents, InputManifest
 from manifests.manifests import Manifests
 from manifests_workflow.component_opensearch import ComponentOpenSearch
@@ -29,6 +31,10 @@ def __init__(self, name: str) -> None:
     def manifests_path(self) -> str:
         return os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "..", "manifests"))
 
+    @classmethod
+    def workflows_path(self) -> str:
+        return os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "..", ".github", "workflows"))
+
     @classmethod
     def legacy_manifests_path(self) -> str:
         return os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "..", "legacy-manifests"))
@@ -41,6 +47,10 @@ def jenkins_path(self) -> str:
     def cron_jenkinsfile(self) -> str:
         return os.path.join(self.jenkins_path(), "check-for-build.jenkinsfile")
 
+    @classmethod
+    def versionincrement_workflow(self) -> str:
+        return os.path.join(self.workflows_path(), "increment-plugin-versions.yml")
+
     @classmethod
     def files(self, name: str) -> List:
         results = []
@@ -114,6 +124,7 @@ def update(
             for release_version in sorted(main_versions.keys() - known_versions):
                 self.write_manifest(release_version, main_versions[release_version])
                 self.add_to_cron(release_version)
+                self.add_to_versionincrement_workflow(release_version)
 
     def create_manifest(self, version: str, components: List = []) -> InputManifest:
         templates_base_path = os.path.join(self.manifests_path(), "templates")
@@ -160,3 +171,28 @@ def add_to_cron(self, version: str) -> None:
             f.write(data)
 
         logging.info(f"Wrote {jenkinsfile}")
+
+    def add_to_versionincrement_workflow(self, version: str) -> None:
+        logging.info("Adding new version the version increment workflow")
+        versionincrement_workflow_file = self.versionincrement_workflow()
+        yaml = ruamel.yaml.YAML()
+        yaml.explicit_start = True
+        yaml.preserve_quotes = True
+
+        with open(versionincrement_workflow_file) as f:
+            data = yaml.load(f)
+
+        version_entry = version.split(".")[0] + ".x"
+        for branch in data["jobs"]["plugin-version-increment-sync"]["strategy"]["matrix"]["branch"]:
+            if version_entry in branch:
+                print(f"{versionincrement_workflow_file} already contains an entry for {version_entry}")
+                return None
+
+        branch_list = list(data["jobs"]["plugin-version-increment-sync"]["strategy"]["matrix"]["branch"])
+        branch_list.append(version_entry)
+        data["jobs"]["plugin-version-increment-sync"]["strategy"]["matrix"]["branch"] = branch_list
+        yaml.indent(mapping=2, sequence=4, offset=2)
+        with open(versionincrement_workflow_file, 'w') as f:
+            yaml.dump(data, f)
+
+        logging.info("Added new version the version increment workflow")
diff --git a/tests/tests_manifests_workflow/test_input_manifests.py b/tests/tests_manifests_workflow/test_input_manifests.py
@@ -6,6 +6,7 @@
 
 import os
 import unittest
+from typing import Any
 from unittest.mock import MagicMock, call, mock_open, patch
 
 from manifests_workflow.input_manifests import InputManifests
@@ -16,6 +17,10 @@ def test_manifests_path(self) -> None:
         path = os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "..", "manifests"))
         self.assertEqual(path, InputManifests.manifests_path())
 
+    def test_workflows_path(self) -> None:
+        path = os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "..", ".github", "workflows"))
+        self.assertEqual(path, InputManifests.workflows_path())
+
     def test_legacy_manifests_path(self) -> None:
         path = os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "..", "legacy-manifests"))
         self.assertEqual(path, InputManifests.legacy_manifests_path())
@@ -122,5 +127,17 @@ def test_add_to_cron(self, mock_open: MagicMock) -> None:
             f"parameterizedCron '''\n{' ' * 12}H 1 * * * %INPUT_MANIFEST=0.1.2/test-0.1.2.yml;TARGET_JOB_NAME=distribution-build-test\n"
         )
 
+    def test_versionincrement_workflow(self) -> None:
+        self.assertEqual(
+            InputManifests.versionincrement_workflow(),
+            os.path.realpath(os.path.join(os.path.dirname(__file__), "..", "..", ".github", "workflows", "increment-plugin-versions.yml"))
+        )
+
+    @patch("builtins.open", new_callable=mock_open)
+    @patch("manifests_workflow.input_manifests.InputManifests.add_to_versionincrement_workflow")
+    def test_add_to_versionincrement_workflow(self, *mocks: Any) -> None:
+        input_manifests = InputManifests("test")
+        input_manifests.add_to_versionincrement_workflow('0.1.2')
+
     def test_create_manifest_with_components(self) -> None:
         pass
diff --git a/tests/tests_manifests_workflow/test_input_manifests_opensearch.py b/tests/tests_manifests_workflow/test_input_manifests_opensearch.py
@@ -33,13 +33,14 @@ def test_files(self) -> None:
     @patch("manifests_workflow.input_manifests.InputComponents")
     @patch("manifests_workflow.input_manifests.InputManifest.from_file")
     @patch("manifests_workflow.input_manifests.InputManifests.add_to_cron")
+    @patch("manifests_workflow.input_manifests.InputManifests.add_to_versionincrement_workflow")
     @patch("manifests_workflow.input_manifests.InputManifest.from_path")
     @patch("manifests_workflow.input_manifests_opensearch.ComponentOpenSearchMin")
     @patch("manifests_workflow.input_manifests_opensearch.ComponentOpenSearch")
     @patch("manifests_workflow.input_manifests.InputManifest")
     def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch: MagicMock,
                     mock_component_opensearch_min: MagicMock, mock_input_manifest_from_path: MagicMock,
-                    mock_add_to_cron: MagicMock, mock_input_manifest_from_file: MagicMock,
+                    mock_add_to_cron: MagicMock, mock_add_to_versionincrement_workflow: MagicMock, mock_input_manifest_from_file: MagicMock,
                     mock_input_manifest_component: MagicMock, *mocks: MagicMock) -> None:
         mock_component_opensearch_min.return_value = MagicMock(name="OpenSearch")
         mock_component_opensearch_min.branches.return_value = ["main", "0.9.0"]
@@ -73,15 +74,20 @@ def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch:
             call('0.10.0'),
             call('0.9.0')
         ])
+        mock_add_to_versionincrement_workflow.assert_has_calls([
+            call('0.10.0'),
+            call('0.9.0')
+        ])
 
     @patch("os.makedirs")
     @patch("os.chdir")
     @patch("manifests_workflow.input_manifests.InputManifests.add_to_cron")
+    @patch("manifests_workflow.input_manifests.InputManifests.add_to_versionincrement_workflow")
     @patch("manifests_workflow.input_manifests_opensearch.ComponentOpenSearchMin")
     @patch("manifests_workflow.input_manifests_opensearch.ComponentOpenSearch")
     @patch("manifests_workflow.input_manifests_opensearch.InputManifestsOpenSearch.write_manifest")
     def test_update_with_latest_manifest(self, mock_write_manifest: MagicMock, mock_component_opensearch: MagicMock,
-                                         mock_component_opensearch_min: MagicMock, mock_add_to_cron: MagicMock,
+                                         mock_component_opensearch_min: MagicMock, mock_add_to_cron: MagicMock, mock_add_to_versionincrement_workflow: MagicMock,
                                          *mocks: MagicMock) -> None:
         mock_component_opensearch_min.return_value = MagicMock(name="OpenSearch")
         mock_component_opensearch_min.branches.return_value = ["main"]
@@ -93,3 +99,4 @@ def test_update_with_latest_manifest(self, mock_write_manifest: MagicMock, mock_
         mock_component_opensearch_min.branches.assert_called()
         mock_write_manifest.assert_called_with("0.9.0", [mock_component_opensearch_min.checkout.return_value])
         mock_add_to_cron.assert_called_with("0.9.0")
+        mock_add_to_versionincrement_workflow.assert_called_with("0.9.0")
diff --git a/tests/tests_manifests_workflow/test_input_manifests_opensearch_dashboards.py b/tests/tests_manifests_workflow/test_input_manifests_opensearch_dashboards.py
@@ -32,11 +32,12 @@ def test_files(self) -> None:
     @patch("manifests_workflow.input_manifests.InputComponents")
     @patch("manifests_workflow.input_manifests.InputManifest.from_file")
     @patch("manifests_workflow.input_manifests.InputManifests.add_to_cron")
+    @patch("manifests_workflow.input_manifests.InputManifests.add_to_versionincrement_workflow")
     @patch("manifests_workflow.input_manifests.InputManifest.from_path")
     @patch("manifests_workflow.input_manifests_opensearch_dashboards.ComponentOpenSearchDashboardsMin")
     @patch("manifests_workflow.input_manifests.InputManifest")
     def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch_min: MagicMock,
-                    mock_input_manifest_from_path: MagicMock, mock_add_to_cron: MagicMock,
+                    mock_input_manifest_from_path: MagicMock, mock_add_to_cron: MagicMock, mock_add_to_versionincrement_workflow: MagicMock,
                     mock_input_manifest_from_file: MagicMock, mock_input_manifest_component: MagicMock,
                     *mocks: MagicMock) -> None:
         mock_component_opensearch_min.return_value = MagicMock(name="OpenSearch-Dashboards")
@@ -60,3 +61,6 @@ def test_update(self, mock_input_manifest: MagicMock, mock_component_opensearch_
         mock_add_to_cron.assert_has_calls([
             call('0.9.0')
         ])
+        mock_add_to_versionincrement_workflow.assert_has_calls([
+            call('0.9.0')
+        ])