Add performance test scripts

Pipfile

@@ -16,6 +16,7 @@ pytest = "*"
 coverage = "~=4.5.4"
 pytest-cov = "~=2.10.0"
 jproperties = "~=2.1.1"
+retry = "~=0.9"
 sortedcontainers = "*"
 cerberus = "~=1.3.4"
 psutil = "~=5.8"

jenkins/opensearch/perf-test.jenkinsfile

@@ -0,0 +1,224 @@
+lib = library(identifier: "jenkins@20211118", retriever: legacySCM(scm))
+
+pipeline {
+    agent none
+    options {
+        timeout(time: 10, unit: 'HOURS')
+    }
+    environment {
+        AGENT_LABEL = 'Jenkins-Agent-al2-x64-c54xlarge-Docker-Host'
+        AGENT_IMAGE = 'opensearchstaging/ci-runner:ci-runner-centos7-v1'
+        BUNDLE_MANIFEST = 'bundle-manifest.yml'
+    }
+    parameters {
+        string(
+            name: 'GITHUB_TOKEN',
+            description: 'Github token for account access.',
+            trim: true
+        )
+        string(
+            name: 'BUNDLE_MANIFEST_URL',
+            description: 'The bundle manifest URL, e.g. https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.2.2/98/linux/x64/builds/opensearch/manifest.yml.',
+            trim: true
+        )
+        string(
+            defaultValue: 'nyc_taxis',
+            name: 'TEST_WORKLOAD',
+            description: 'The workload name from OpenSearch Benchmark Workloads for Mensor (internal client).',
+            trim: true
+        )
+        string(
+            defaultValue: '1',
+            name: 'TEST_ITERATIONS',
+            description: 'Number of times to run a workload for Mensor (internal client).',
+            trim: true
+        )
+        string(
+            defaultValue: '0',
+            name: 'WARMUP_ITERATIONS',
+            description: 'Number of times to run a workload before collecting data for Mensor (internal client).',
+            trim: true
+        )
+    }
+
+    stages {
+        stage('validate-and-set-parameters') {
+            agent {
+                docker {
+                    label AGENT_LABEL
+                    image AGENT_IMAGE
+                    alwaysPull true
+                }
+            }
+            steps {
+                script {
+                    if (BUNDLE_MANIFEST_URL == '') {
+                        currentBuild.result = 'ABORTED'
+                        error("Performance Tests failed to start. Missing parameter: BUNDLE_MANIFEST_URL.")
+                    }
+                    if (GITHUB_TOKEN == '') {
+                        currentBuild.result = 'ABORTED'
+                        error("Performance Tests failed to start. Missing parameter: GITHUB_TOKEN.")
+                    }
+                    if (TEST_ITERATIONS != null && !TEST_ITERATIONS.isInteger()) {
+                        currentBuild.result = 'ABORTED'
+                        error("Performance Tests failed to start. Invalid value for parameter: TEST_ITERATIONS. Value should be an integer.")
+                    }
+                    if (WARMUP_ITERATIONS != null && !WARMUP_ITERATIONS.isInteger()) {
+                        currentBuild.result = 'ABORTED'
+                        error("Performance Tests failed to start. Invalid value for parameter: WARMUP_ITERATIONS. Value should be an integer.")
+                    }
+                    def bundleManifestObj = downloadBuildManifest(
+                        url: BUNDLE_MANIFEST_URL,
+                        path: BUNDLE_MANIFEST
+                    )
+                    String buildId = bundleManifestObj.getArtifactBuildId()
+                    env.BUILD_ID = buildId
+                    env.HAS_SECURITY = bundleManifestObj.components.containsKey("security")
+                    env.ARCHITECTURE = bundleManifestObj.getArtifactArchitecture()
+                }
+            }
+        }
+        stage('perf-test') {
+            parallel {
+                stage('test-with-security') {
+                    agent {
+                        docker {
+                            label AGENT_LABEL
+                            image AGENT_IMAGE
+                            alwaysPull true
+                        }
+                    }
+                    when {
+                        expression { HAS_SECURITY == true }
+                    }
+                    steps {
+                        script {
+                            def bundleManifestObj = downloadBuildManifest(
+                                url: BUNDLE_MANIFEST_URL,
+                                path: BUNDLE_MANIFEST
+                            )
+                            echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}"
+                            echo "BUILD_ID: ${BUILD_ID}"
+                            echo "Architecture: ${ARCHITECTURE}"
+
+                            lib.jenkins.Messages.new(this).add('perf-test', "Performance tests for #${BUILD_ID}")
+
+                            runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}",
+                                buildId: "${BUILD_ID}",
+                                architecture: "${ARCHITECTURE}",
+                                insecure: false,
+                                workload: TEST_WORKLOAD,
+                                testIterations: TEST_ITERATIONS,
+                                warmupIterations: WARMUP_ITERATIONS)
+
+                            lib.jenkins.Messages.new(this).add('perf-test', "Performance tests with security for ${BUILD_ID} completed")
+                        }
+                    }
+                    post {
+                        success {
+                            script {
+                                uploadTestResults(
+                                    buildManifestFileName: "${BUNDLE_MANIFEST}",
+                                    jobName: 'perf-test',
+                                    buildNumber: "${BUILD_ID}"
+                                )
+                            }
+                            postCleanup()
+                        }
+                        failure {
+                            postCleanup()
+                        }
+                        aborted {
+                            postCleanup()
+                        }
+                    }
+                }
+                stage('test-without-security') {
+                    agent {
+                        docker {
+                            label AGENT_LABEL
+                            image AGENT_IMAGE
+                            alwaysPull true
+                        }
+                    }
+                    steps {
+                        script {
+                            def bundleManifestObj = downloadBuildManifest(
+                                url: BUNDLE_MANIFEST_URL,
+                                path: BUNDLE_MANIFEST
+                            )
+
+                            echo "BUNDLE_MANIFEST: ${BUNDLE_MANIFEST}"
+                            echo "BUILD_ID: ${BUILD_ID}"
+                            echo "Architecture: ${ARCHITECTURE}"
+
+                            lib.jenkins.Messages.new(this).add('perf-test', "Performance tests for #${BUILD_ID}")
+
+                            runPerfTestScript(bundleManifest: "${BUNDLE_MANIFEST}",
+                                buildId: "${BUILD_ID}",
+                                architecture: "${ARCHITECTURE}",
+                                insecure: true,
+                                workload: TEST_WORKLOAD,
+                                testIterations: TEST_ITERATIONS,
+                                warmupIterations: WARMUP_ITERATIONS)
+
+                            lib.jenkins.Messages.new(this).add('perf-test', "Performance tests without security for ${BUILD_ID} completed")
+                        }
+                    }
+                    post {
+                        success {
+                            script {
+                                uploadTestResults(
+                                    buildManifestFileName: "${BUNDLE_MANIFEST}",
+                                    jobName: 'perf-test',
+                                    buildNumber: "${BUILD_ID}"
+                                )
+                            }
+                            postCleanup()
+                        }
+                        failure {
+                            postCleanup()
+                        }
+                        aborted {
+                            postCleanup()
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    post {
+        success {
+            node(AGENT_LABEL) {
+                script {
+                    def stashed = lib.jenkins.Messages.new(this).get(['perf-test'])
+                    publishNotification(
+                        icon: ':white_check_mark:',
+                        message: 'Performance Tests Successful',
+                        extra: stashed,
+                        credentialsId: 'INTEG_TEST_WEBHOOK',
+                    )
+
+                    postCleanup()
+                }
+            }
+        }
+        failure {
+            node(AGENT_LABEL) {
+                script {
+                    def stashed = lib.jenkins.Messages.new(this).get(['perf-test'])
+                    publishNotification(
+                        icon: ':warning:',
+                        message: 'Failed Performance Tests',
+                        extra: stashed,
+                        credentialsId: 'INTEG_TEST_WEBHOOK',
+                    )
+
+                    postCleanup()
+                }
+            }
+        }
+    }
+}

src/run_perf_test.py

@@ -11,6 +11,7 @@
 import sys
 
 import yaml
+from retry.api import retry_call
 
 from git.git_repository import GitRepository
 from manifests.bundle_manifest import BundleManifest
@@ -35,6 +36,9 @@ def main():
     parser.add_argument("--bundle-manifest", type=argparse.FileType("r"), help="Bundle Manifest file.", required=True)
     parser.add_argument("--stack", dest="stack", help="Stack name for performance test")
     parser.add_argument("--config", type=argparse.FileType("r"), help="Config file.", required=True)
+    parser.add_argument("--force-insecure-mode", dest="insecure", action="store_true",
+                        help="Force the security of the cluster to be disabled.",
+                        default=False)
     parser.add_argument("--keep", dest="keep", action="store_true", help="Do not delete the working temporary directory.")
     parser.add_argument("--workload", default="nyc_taxis", help="Mensor (internal client) param - Workload name from OpenSeach Benchmark Workloads")
     parser.add_argument("--workload-options", default="{}", help="Mensor (internal client) param - Json object with OpenSearch Benchmark arguments")
@@ -45,14 +49,22 @@ def main():
     manifest = BundleManifest.from_file(args.bundle_manifest)
     config = yaml.safe_load(args.config)
 
+    security = "security" in manifest.components and not args.insecure
+    if security:
+        tests_dir = os.path.join(os.getcwd(), "test-results", "perf-test", "with-security")
+    else:
+        tests_dir = os.path.join(os.getcwd(), "test-results", "perf-test", "without-security")
+    os.makedirs(tests_dir, exist_ok=True)
+
     with TemporaryDirectory(keep=args.keep, chdir=True) as work_dir:
         current_workspace = os.path.join(work_dir.name, "infra")
         with GitRepository(get_infra_repo_url(), "main", current_workspace):
-            security = "security" in manifest.components
             with WorkingDirectory(current_workspace):
-                with PerfTestCluster.create(manifest, config, args.stack, security, current_workspace) as (test_cluster_endpoint, test_cluster_port):
-                    perf_test_suite = PerfTestSuite(manifest, test_cluster_endpoint, security, current_workspace, args)
-                    perf_test_suite.execute()
+                with PerfTestCluster.create(manifest, config, args.stack, security, current_workspace) \
+                        as (test_cluster_endpoint, test_cluster_port):
+                    perf_test_suite = PerfTestSuite(manifest, test_cluster_endpoint, security,
+                                                    current_workspace, tests_dir, args)
+                    retry_call(perf_test_suite.execute, tries=3, delay=60, backoff=2)
 
 
 if __name__ == "__main__":

src/test_workflow/perf_test/perf_test_suite.py

@@ -8,8 +8,7 @@ class PerfTestSuite:
     """
     Represents a performance test suite. This class runs rally test on the deployed cluster with the provided IP.
     """
-
-    def __init__(self, bundle_manifest, endpoint, security, current_workspace, args):
+    def __init__(self, bundle_manifest, endpoint, security, current_workspace, test_results_path, args):
         self.manifest = bundle_manifest
         self.work_dir = "mensor/"
         self.endpoint = endpoint
@@ -23,16 +22,22 @@ def __init__(self, bundle_manifest, endpoint, security, current_workspace, args)
             f" --warmup-iters {self.args.warmup_iters} --test-iters {self.args.test_iters}"
         )
 
+        if test_results_path is not None:
+            self.command = (
+                f"pipenv run python test_config.py -i {self.endpoint} -b {self.manifest.build.id}"
+                f" -a {self.manifest.build.architecture} -p {test_results_path}"
+                f" --workload {self.args.workload} --workload-options '{self.args.workload_options}'"
+                f" --warmup-iters {self.args.warmup_iters} --test-iters {self.args.test_iters}"
+            )
+
     def execute(self):
         try:
-            with WorkingDirectory(self.work_dir):
-                dir = os.getcwd()
-                subprocess.check_call("python3 -m pipenv install", cwd=dir, shell=True)
-                subprocess.check_call("pipenv install", cwd=dir, shell=True)
-
+            current_workspace = os.path.join(self.current_workspace, self.work_dir)
+            with WorkingDirectory(current_workspace):
+                subprocess.check_call("pipenv install", cwd=current_workspace, shell=True)
                 if self.security:
-                    subprocess.check_call(f"{self.command} -s", cwd=dir, shell=True)
+                    subprocess.check_call(f"{self.command} -s", cwd=current_workspace, shell=True)
                 else:
-                    subprocess.check_call(f"{self.command}", cwd=dir, shell=True)
+                    subprocess.check_call(f"{self.command}", cwd=current_workspace, shell=True)
         finally:
             os.chdir(self.current_workspace)

test.sh

@@ -16,8 +16,11 @@ case $1 in
   "bwc-test")
   "$DIR/run.sh" "$DIR/src/run_bwc_test.py" "${@:2}"
   ;;
+  "perf-test")
+  "$DIR/run.sh" "$DIR/src/run_perf_test.py" "${@:2}"
+  ;;
   *)
-  echo "Invalid test suite, run ./test.sh integ-test|bwc-test."
+  echo "Invalid test suite, run ./test.sh integ-test|bwc-test|perf-test."
   exit 1
   ;;
 esac

tests/data/perf-test-config.yml

@@ -0,0 +1,9 @@
+---
+Description: Configuration file to store contants required to run rally for performance test
+Constants:
+  Repository: https://github.com/opensearch-project/
+  SecurityGroupId: test-security
+  VpcId: test-vpc
+  AccountId: 123456
+  Region: eu-east-1
+  Role: test-role-name