From 81e1fd2f1b958fa5fd001b2c78fd35c3ff322232 Mon Sep 17 00:00:00 2001 From: Brian Seeders Date: Fri, 28 Jul 2023 14:49:46 -0400 Subject: [PATCH] Add java-matrix and java-fips-matrix Buildkite pipelines (#97253) --- .buildkite/hooks/pre-command | 45 ++++++++++++++++ .buildkite/pipelines/periodic.yml | 52 +++++++++++++++++++ .ci/init.gradle | 30 +++++++---- .ci/scripts/packaging-test.sh | 1 + .ci/scripts/run-gradle.sh | 1 + .../groovy/elasticsearch.build-scan.gradle | 2 +- catalog-info.yaml | 47 +++++++++++++++-- distribution/packages/build.gradle | 1 + 8 files changed, 166 insertions(+), 13 deletions(-) create mode 100644 .buildkite/hooks/pre-command create mode 100644 .buildkite/pipelines/periodic.yml diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command new file mode 100644 index 0000000000000..e40abefbe7632 --- /dev/null +++ b/.buildkite/hooks/pre-command @@ -0,0 +1,45 @@ +#!/bin/bash + +# On some distros, this directory ends up not readable by the `elasticsearch` user that gets created during tests +# This fixes that +chmod 755 ~ + +WORKSPACE="$(pwd)" +export WORKSPACE + +BUILD_NUMBER="$BUILDKITE_BUILD_NUMBER" +export BUILD_NUMBER + +COMPOSE_HTTP_TIMEOUT="120" +export COMPOSE_HTTP_TIMEOUT + +JOB_BRANCH="$BUILDKITE_BRANCH" +export JOB_BRANCH + +GRADLEW="./gradlew --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/" +export GRADLEW + +GRADLEW_BAT="./gradlew.bat --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/" +export GRADLEW_BAT + +export $(cat .ci/java-versions.properties | grep '=' | xargs) + +JAVA_HOME="$HOME/.java/$ES_BUILD_JAVA" +export JAVA_HOME + +JAVA11_HOME="$HOME/.java/java11" +export JAVA11_HOME + +JAVA16_HOME="$HOME/.java/openjdk16" +export JAVA16_HOME + +if [[ "${ES_RUNTIME_JAVA:-}" ]]; then + RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA + export RUNTIME_JAVA_HOME +fi + +GRADLE_BUILD_CACHE_USERNAME=$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache) +export GRADLE_BUILD_CACHE_USERNAME + +GRADLE_BUILD_CACHE_PASSWORD=$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache) +export GRADLE_BUILD_CACHE_PASSWORD diff --git a/.buildkite/pipelines/periodic.yml b/.buildkite/pipelines/periodic.yml new file mode 100644 index 0000000000000..e2368a0bee2da --- /dev/null +++ b/.buildkite/pipelines/periodic.yml @@ -0,0 +1,52 @@ +steps: + - group: java-fips-matrix + steps: + - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-fips-matrix" + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.fips.enabled=true $$GRADLE_TASK + timeout_in_minutes: 180 + matrix: + setup: + ES_RUNTIME_JAVA: + - openjdk17 + GRADLE_TASK: + - checkPart1 + - checkPart2 + - checkPart3 + - bwcTestSnapshots + - checkRestCompat + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + env: + ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" + GRADLE_TASK: "{{matrix.GRADLE_TASK}}" + - group: java-matrix + steps: + - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-matrix" + command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true $$GRADLE_TASK + timeout_in_minutes: 180 + matrix: + setup: + ES_RUNTIME_JAVA: + - graalvm-ce17 + - openjdk17 + - openjdk18 + - openjdk19 + - openjdk20 + - openjdk21 + GRADLE_TASK: + - checkPart1 + - checkPart2 + - checkPart3 + - bwcTestSnapshots + - checkRestCompat + agents: + provider: gcp + image: family/elasticsearch-ubuntu-2004 + machineType: custom-32-98304 + buildDirectory: /dev/shm/bk + env: + ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}" + GRADLE_TASK: "{{matrix.GRADLE_TASK}}" diff --git a/.ci/init.gradle b/.ci/init.gradle index ce5ee20f8fbd8..4b2cbd1907ca0 100644 --- a/.ci/init.gradle +++ b/.ci/init.gradle @@ -13,14 +13,26 @@ initscript { boolean USE_ARTIFACTORY = false if (System.getenv('VAULT_ADDR') == null) { + // When trying to reproduce errors outside of CI, it can be useful to allow this to just return rather than blowing up + if (System.getenv('CI') == null) { + return + } + throw new GradleException("You must set the VAULT_ADDR environment variable to use this init script.") } if (System.getenv('VAULT_ROLE_ID') == null && System.getenv('VAULT_SECRET_ID') == null && System.getenv('VAULT_TOKEN') == null) { + // When trying to reproduce errors outside of CI, it can be useful to allow this to just return rather than blowing up + if (System.getenv('CI') == null) { + return + } + throw new GradleException("You must set either the VAULT_ROLE_ID and VAULT_SECRET_ID environment variables, " + "or the VAULT_TOKEN environment variable to use this init script.") } +final String vaultPathPrefix = System.getenv('VAULT_ADDR') ==~ /.+vault-ci.+\.dev.*/ ? "secret/ci/elastic-elasticsearch/migrated" : "secret/elasticsearch-ci" + final String vaultToken = System.getenv('VAULT_TOKEN') ?: new Vault( new VaultConfig() .address(System.env.VAULT_ADDR) @@ -44,7 +56,7 @@ final Vault vault = new Vault( if (USE_ARTIFACTORY) { final Map artifactoryCredentials = vault.logical() - .read("secret/elasticsearch-ci/artifactory.elstc.co") + .read("${vaultPathPrefix}/artifactory.elstc.co") .getData() logger.info("Using elastic artifactory repos") Closure configCache = { @@ -78,10 +90,10 @@ if (USE_ARTIFACTORY) { } } -projectsLoaded { - rootProject { - project.pluginManager.withPlugin('com.gradle.build-scan') { - buildScan.server = 'https://gradle-enterprise.elastic.co' +gradle.settingsEvaluated { settings -> + settings.pluginManager.withPlugin("com.gradle.enterprise") { + settings.gradleEnterprise { + server = 'https://gradle-enterprise.elastic.co' } } } @@ -91,8 +103,8 @@ final String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.u final boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false')) if (buildCacheUrl) { - final Map buildCacheCredentials = vault.logical() - .read("secret/elasticsearch-ci/gradle-build-cache") + final Map buildCacheCredentials = System.getenv("GRADLE_BUILD_CACHE_USERNAME") ? [:] : vault.logical() + .read("${vaultPathPrefix}/gradle-build-cache") .getData() gradle.settingsEvaluated { settings -> settings.buildCache { @@ -104,8 +116,8 @@ if (buildCacheUrl) { url = buildCacheUrl push = buildCachePush credentials { - username = buildCacheCredentials.get("username") - password = buildCacheCredentials.get("password") + username = System.getenv("GRADLE_BUILD_CACHE_USERNAME") ?: buildCacheCredentials.get("username") + password = System.getenv("GRADLE_BUILD_CACHE_PASSWORD") ?: buildCacheCredentials.get("password") } } } diff --git a/.ci/scripts/packaging-test.sh b/.ci/scripts/packaging-test.sh index d1e17d08bf762..afe162b23e564 100755 --- a/.ci/scripts/packaging-test.sh +++ b/.ci/scripts/packaging-test.sh @@ -39,6 +39,7 @@ if [ -f "/etc/os-release" ] ; then # Work around incorrect lintian version # https://github.com/elastic/elasticsearch/issues/48573 if [ $VERSION_ID == 10 ] ; then + sudo apt-get update -y sudo apt-get install -y --allow-downgrades lintian=2.15.0 fi fi diff --git a/.ci/scripts/run-gradle.sh b/.ci/scripts/run-gradle.sh index a63895465d52f..2c59817929443 100755 --- a/.ci/scripts/run-gradle.sh +++ b/.ci/scripts/run-gradle.sh @@ -1,6 +1,7 @@ #!/bin/bash # drop page cache and kernel slab objects on linux [[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches + rm -Rfv ~/.gradle/init.d mkdir -p ~/.gradle/init.d && cp -v $WORKSPACE/.ci/init.gradle ~/.gradle/init.d if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then diff --git a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle index 575d514126765..cd488e0dc0d19 100644 --- a/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle +++ b/build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle @@ -16,7 +16,7 @@ buildScan { String buildKiteUrl = System.getenv('BUILDKITE_BUILD_URL') ? System.getenv('BUILDKITE_BUILD_URL') : null // Automatically publish scans from Elasticsearch CI - if (jenkinsUrl?.host?.endsWith('elastic.co') || jenkinsUrl?.host?.endsWith('elastic.dev')) { + if (jenkinsUrl?.host?.endsWith('elastic.co') || jenkinsUrl?.host?.endsWith('elastic.dev') || System.getenv('BUILDKITE') == 'true') { publishAlways() buildScan.server = 'https://gradle-enterprise.elastic.co' } diff --git a/catalog-info.yaml b/catalog-info.yaml index 38b7752a893d8..a6e27bbbacf03 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -16,13 +16,14 @@ spec: apiVersion: buildkite.elastic.dev/v1 kind: Pipeline metadata: - description: ':elasticsearch: Update elasticsearch submodule in elasticsearch-serverless' + description: ":elasticsearch: Update elasticsearch submodule in elasticsearch-serverless" name: elasticsearch / update serverless submodule spec: repository: elastic/elasticsearch pipeline_file: .buildkite/update-es-serverless.yml teams: elasticsearch-team: {} + ml-core: {} everyone: access_level: READ_ONLY provider_settings: @@ -30,7 +31,7 @@ spec: schedules: daily promotion: branch: main - cronline: '@daily' + cronline: "@daily" --- # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json apiVersion: backstage.io/v1alpha1 @@ -49,7 +50,7 @@ spec: apiVersion: buildkite.elastic.dev/v1 kind: Pipeline metadata: - description: ':elasticsearch: Validate elasticsearch changes against serverless' + description: ":elasticsearch: Validate elasticsearch changes against serverless" name: elasticsearch / check serverless submodule spec: repository: elastic/elasticsearch @@ -57,8 +58,48 @@ spec: branch_configuration: main teams: elasticsearch-team: {} + ml-core: {} everyone: access_level: READ_ONLY provider_settings: build_pull_requests: false publish_commit_status: false +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: buildkite-pipeline-elasticsearch-periodic + description: Elasticsearch tests and checks that are run a few times daily + links: + - title: Pipeline + url: https://buildkite.com/elastic/elasticsearch-periodic +spec: + type: buildkite-pipeline + system: buildkite + owner: group:elasticsearch-team + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + description: ":elasticsearch: Tests and checks that are run a few times daily" + name: elasticsearch / periodic + spec: + repository: elastic/elasticsearch + pipeline_file: .buildkite/pipelines/periodic.yml + branch_configuration: main + teams: + elasticsearch-team: {} + ml-core: {} + everyone: + access_level: READ_ONLY + provider_settings: + build_branches: false + build_pull_requests: false + publish_commit_status: false + trigger_mode: none + schedules: + Periodically on main: + branch: main + cronline: "0 0,8,16 * * America/New_York" + message: "Tests and checks that are run 3x daily" diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle index c5c6528272621..c99a9af302c40 100644 --- a/distribution/packages/build.gradle +++ b/distribution/packages/build.gradle @@ -308,6 +308,7 @@ Closure commonDebConfig(String architecture) { into('/usr/share/lintian/overrides') { from('src/deb/lintian/elasticsearch') + fileMode 0644 } } }