From 639384e35104773a6e215af735b0fa221cef2e6b Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Tue, 27 Aug 2024 16:41:39 -0400 Subject: [PATCH 1/2] Enable check-for-build to trigger linux-tar integTest during distribution build cron Signed-off-by: Peter Zhu --- jenkins/check-for-build.jenkinsfile | 20 ++++++++++--- src/manifests_workflow/input_manifests.py | 30 +++++++++++-------- .../test_input_manifests.py | 5 ++-- 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/jenkins/check-for-build.jenkinsfile b/jenkins/check-for-build.jenkinsfile index cefe5a265b..d9e19941e4 100644 --- a/jenkins/check-for-build.jenkinsfile +++ b/jenkins/check-for-build.jenkinsfile @@ -23,10 +23,10 @@ pipeline { } triggers { parameterizedCron ''' - H 1 * * * %INPUT_MANIFEST=2.16.1/opensearch-2.16.1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip - H 1 * * * %INPUT_MANIFEST=2.17.0/opensearch-2.17.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip - H 1 * * * %INPUT_MANIFEST=2.17.0/opensearch-dashboards-2.17.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip - H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux macos windows;BUILD_DISTRIBUTION=tar rpm deb zip + H 1 * * * %INPUT_MANIFEST=2.16.1/opensearch-2.16.1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip + H 1 * * * %INPUT_MANIFEST=2.17.0/opensearch-2.17.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip;TEST_MANIFEST=2.17.0/opensearch-2.17.0-test.yml;TEST_PLATFORM=linux;TEST_DISTRIBUTION=tar + H 1 * * * %INPUT_MANIFEST=2.17.0/opensearch-dashboards-2.17.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip;TEST_MANIFEST=2.17.0/opensearch-2.17.0-test.yml;TEST_PLATFORM=linux;TEST_DISTRIBUTION=tar + H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-dashboards-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip ''' } @@ -56,6 +56,16 @@ pipeline { description: 'Distribution to build', trim: true ) + string( + name: 'TEST_PLATFORM', + description: 'Platform to test', + trim: true + ) + string( + name: 'TEST_DISTRIBUTION', + description: 'Distribution to test', + trim: true + ) } stages { stage('detect docker image + args') { @@ -102,6 +112,8 @@ pipeline { string(name: 'TEST_MANIFEST', value: "${TEST_MANIFEST}"), string(name: 'BUILD_PLATFORM', value: "${BUILD_PLATFORM}"), string(name: 'BUILD_DISTRIBUTION', value: "${BUILD_DISTRIBUTION}") + string(name: 'TEST_PLATFORM', value: "${TEST_PLATFORM}"), + string(name: 'TEST_DISTRIBUTION', value: "${TEST_DISTRIBUTION}") ], wait: true echo "Build succeeded, uploading build SHA for that job" diff --git a/src/manifests_workflow/input_manifests.py b/src/manifests_workflow/input_manifests.py index 1d411a2477..187fc200ee 100644 --- a/src/manifests_workflow/input_manifests.py +++ b/src/manifests_workflow/input_manifests.py @@ -25,15 +25,6 @@ class InputManifests(Manifests): - BUILD_PLATFORM = { - "opensearch": "linux macos windows", - "opensearch-dashboards": "linux windows" - } - BUILD_DISTRIBUTION = { - "opensearch": "tar rpm deb zip", - "opensearch-dashboards": "tar rpm deb zip" - } - def __init__(self, name: str) -> None: self.name = name self.prefix = name.lower().replace(" ", "-") @@ -106,6 +97,7 @@ def update( # check out and build #main, 1.x, etc. all_branches = sorted(min_klass.branches()) branches = [b for b in all_branches if not any(b == o or b.startswith((f"{o}-", f"{o}/")) for o in legacy_branches)] + # TODO: Remove logging.info(f"Checking {self.name} {sorted(branches)} branches") logging.info(f"Ignoring {self.name} {sorted(set(all_branches) - set(branches))} branches as they are legacy") @@ -176,13 +168,27 @@ def add_to_cron(self, version: str) -> None: with open(jenkinsfile, "r") as f: data = f.read() - build_platform = self.BUILD_PLATFORM.get(self.prefix, "linux") - build_distribution = self.BUILD_DISTRIBUTION.get(self.prefix, "tar") + # Note: default to linux tar for now as integTest is very heavy on resources + build_platform_map = { + "opensearch": "linux windows", + "opensearch-dashboards": "linux windows" + } + build_distribution_map = { + "opensearch": "tar rpm deb zip", + "opensearch-dashboards": "tar rpm deb zip" + } + build_platform = build_platform_map.get(self.prefix, "linux") + build_distribution = build_distribution_map.get(self.prefix, "tar") + test_platform = "linux" + test_distribution = "tar" cron_entry = f"H 1 * * * %INPUT_MANIFEST={version}/{self.prefix}-{version}.yml;" \ f"TARGET_JOB_NAME=distribution-build-{self.prefix};" \ f"BUILD_PLATFORM={build_platform};" \ - f"BUILD_DISTRIBUTION={build_distribution}\n" + f"BUILD_DISTRIBUTION={build_distribution};" \ + f"TEST_MANIFEST={version}/{self.prefix}-{version}-test.yml;" \ + f"TEST_PLATFORM={test_platform};" \ + f"TEST_DISTRIBUTION={test_distribution}\n" if cron_entry in data: raise ValueError(f"{jenkinsfile} already contains an entry for {self.prefix} {version}") diff --git a/tests/tests_manifests_workflow/test_input_manifests.py b/tests/tests_manifests_workflow/test_input_manifests.py index 17e32cbb55..be5aad53fa 100644 --- a/tests/tests_manifests_workflow/test_input_manifests.py +++ b/tests/tests_manifests_workflow/test_input_manifests.py @@ -154,8 +154,9 @@ def test_add_to_cron(self, mock_open: MagicMock) -> None: mock_open.assert_has_calls([call(InputManifests.cron_jenkinsfile(), 'w')]) mock_open.assert_has_calls([call(InputManifests.cron_jenkinsfile(), 'r')]) mock_open().write.assert_called_once_with( - f"parameterizedCron '''\n{' ' * 12}H 1 * * * %INPUT_MANIFEST=0.1.2/test-0.1.2.yml;" - f"TARGET_JOB_NAME=distribution-build-test;BUILD_PLATFORM=linux;BUILD_DISTRIBUTION=tar\n" + "parameterizedCron '''\n H 1 * * * %INPUT_MANIFEST=0.1.2/test-0.1.2.yml;" + "TARGET_JOB_NAME=distribution-build-test;BUILD_PLATFORM=linux;BUILD_DISTRIBUTION=tar;" + "TEST_MANIFEST=0.1.2/test-0.1.2-test.yml;TEST_PLATFORM=linux;TEST_DISTRIBUTION=tar\n" ) def test_os_versionincrement_workflow(self) -> None: From cbd1722508a7ac3627463f4cc6ce253ec225ec6a Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 28 Aug 2024 14:09:05 -0400 Subject: [PATCH 2/2] small tweaks Signed-off-by: Peter Zhu --- jenkins/check-for-build.jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/check-for-build.jenkinsfile b/jenkins/check-for-build.jenkinsfile index d9e19941e4..7df17a8a03 100644 --- a/jenkins/check-for-build.jenkinsfile +++ b/jenkins/check-for-build.jenkinsfile @@ -25,7 +25,7 @@ pipeline { parameterizedCron ''' H 1 * * * %INPUT_MANIFEST=2.16.1/opensearch-2.16.1.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip H 1 * * * %INPUT_MANIFEST=2.17.0/opensearch-2.17.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip;TEST_MANIFEST=2.17.0/opensearch-2.17.0-test.yml;TEST_PLATFORM=linux;TEST_DISTRIBUTION=tar - H 1 * * * %INPUT_MANIFEST=2.17.0/opensearch-dashboards-2.17.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip;TEST_MANIFEST=2.17.0/opensearch-2.17.0-test.yml;TEST_PLATFORM=linux;TEST_DISTRIBUTION=tar + H 1 * * * %INPUT_MANIFEST=2.17.0/opensearch-dashboards-2.17.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip;TEST_MANIFEST=2.17.0/opensearch-dashboards-2.17.0-test.yml;TEST_PLATFORM=linux;TEST_DISTRIBUTION=tar H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip H 1 * * * %INPUT_MANIFEST=3.0.0/opensearch-dashboards-3.0.0.yml;TARGET_JOB_NAME=distribution-build-opensearch-dashboards;BUILD_PLATFORM=linux windows;BUILD_DISTRIBUTION=tar rpm deb zip '''