Add java-matrix and java-fips-matrix Buildkite pipelines (elastic#97253)

.buildkite/hooks/pre-command

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# On some distros, this directory ends up not readable by the `elasticsearch` user that gets created during tests
+# This fixes that
+chmod 755 ~
+
+WORKSPACE="$(pwd)"
+export WORKSPACE
+
+BUILD_NUMBER="$BUILDKITE_BUILD_NUMBER"
+export BUILD_NUMBER
+
+COMPOSE_HTTP_TIMEOUT="120"
+export COMPOSE_HTTP_TIMEOUT
+
+JOB_BRANCH="$BUILDKITE_BRANCH"
+export JOB_BRANCH
+
+GRADLEW="./gradlew --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
+export GRADLEW
+
+GRADLEW_BAT="./gradlew.bat --parallel --scan --build-cache --no-watch-fs -Dorg.elasticsearch.build.cache.url=https://gradle-enterprise.elastic.co/cache/"
+export GRADLEW_BAT
+
+export $(cat .ci/java-versions.properties | grep '=' | xargs)
+
+JAVA_HOME="$HOME/.java/$ES_BUILD_JAVA"
+export JAVA_HOME
+
+JAVA11_HOME="$HOME/.java/java11"
+export JAVA11_HOME
+
+JAVA16_HOME="$HOME/.java/openjdk16"
+export JAVA16_HOME
+
+if [[ "${ES_RUNTIME_JAVA:-}" ]]; then
+  RUNTIME_JAVA_HOME=$HOME/.java/$ES_RUNTIME_JAVA
+  export RUNTIME_JAVA_HOME
+fi
+
+GRADLE_BUILD_CACHE_USERNAME=$(vault read -field=username secret/ci/elastic-elasticsearch/migrated/gradle-build-cache)
+export GRADLE_BUILD_CACHE_USERNAME
+
+GRADLE_BUILD_CACHE_PASSWORD=$(vault read -field=password secret/ci/elastic-elasticsearch/migrated/gradle-build-cache)
+export GRADLE_BUILD_CACHE_PASSWORD

.buildkite/pipelines/periodic.yml

@@ -0,0 +1,52 @@
+steps:
+  - group: java-fips-matrix
+    steps:
+      - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-fips-matrix"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.fips.enabled=true $$GRADLE_TASK
+        timeout_in_minutes: 180
+        matrix:
+          setup:
+            ES_RUNTIME_JAVA:
+              - openjdk17
+            GRADLE_TASK:
+              - checkPart1
+              - checkPart2
+              - checkPart3
+              - bwcTestSnapshots
+              - checkRestCompat
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
+          GRADLE_TASK: "{{matrix.GRADLE_TASK}}"
+  - group: java-matrix
+    steps:
+      - label: "{{matrix.ES_RUNTIME_JAVA}} / {{matrix.GRADLE_TASK}} / java-matrix"
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true $$GRADLE_TASK
+        timeout_in_minutes: 180
+        matrix:
+          setup:
+            ES_RUNTIME_JAVA:
+              - graalvm-ce17
+              - openjdk17
+              - openjdk18
+              - openjdk19
+              - openjdk20
+              - openjdk21
+            GRADLE_TASK:
+              - checkPart1
+              - checkPart2
+              - checkPart3
+              - bwcTestSnapshots
+              - checkRestCompat
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2004
+          machineType: custom-32-98304
+          buildDirectory: /dev/shm/bk
+        env:
+          ES_RUNTIME_JAVA: "{{matrix.ES_RUNTIME_JAVA}}"
+          GRADLE_TASK: "{{matrix.GRADLE_TASK}}"

.ci/init.gradle

@@ -13,14 +13,26 @@ initscript {
 boolean USE_ARTIFACTORY = false
 
 if (System.getenv('VAULT_ADDR') == null) {
+  // When trying to reproduce errors outside of CI, it can be useful to allow this to just return rather than blowing up
+  if (System.getenv('CI') == null) {
+    return
+  }
+
   throw new GradleException("You must set the VAULT_ADDR environment variable to use this init script.")
 }
 
 if (System.getenv('VAULT_ROLE_ID') == null && System.getenv('VAULT_SECRET_ID') == null && System.getenv('VAULT_TOKEN') == null) {
+    // When trying to reproduce errors outside of CI, it can be useful to allow this to just return rather than blowing up
+  if (System.getenv('CI') == null) {
+    return
+  }
+
   throw new GradleException("You must set either the VAULT_ROLE_ID and VAULT_SECRET_ID environment variables, " +
     "or the VAULT_TOKEN environment variable to use this init script.")
 }
 
+final String vaultPathPrefix = System.getenv('VAULT_ADDR') ==~ /.+vault-ci.+\.dev.*/ ? "secret/ci/elastic-elasticsearch/migrated" : "secret/elasticsearch-ci"
+
 final String vaultToken = System.getenv('VAULT_TOKEN') ?: new Vault(
   new VaultConfig()
     .address(System.env.VAULT_ADDR)
@@ -44,7 +56,7 @@ final Vault vault = new Vault(
 
 if (USE_ARTIFACTORY) {
   final Map<String, String> artifactoryCredentials = vault.logical()
-    .read("secret/elasticsearch-ci/artifactory.elstc.co")
+    .read("${vaultPathPrefix}/artifactory.elstc.co")
     .getData()
   logger.info("Using elastic artifactory repos")
   Closure configCache = {
@@ -78,10 +90,10 @@ if (USE_ARTIFACTORY) {
   }
 }
 
-projectsLoaded {
-  rootProject {
-    project.pluginManager.withPlugin('com.gradle.build-scan') {
-      buildScan.server = 'https://gradle-enterprise.elastic.co'
+gradle.settingsEvaluated { settings ->
+  settings.pluginManager.withPlugin("com.gradle.enterprise") {
+    settings.gradleEnterprise {
+      server = 'https://gradle-enterprise.elastic.co'
     }
   }
 }
@@ -91,8 +103,8 @@ final String buildCacheUrl = System.getProperty('org.elasticsearch.build.cache.u
 final boolean buildCachePush = Boolean.valueOf(System.getProperty('org.elasticsearch.build.cache.push', 'false'))
 
 if (buildCacheUrl) {
-  final Map<String, String> buildCacheCredentials = vault.logical()
-    .read("secret/elasticsearch-ci/gradle-build-cache")
+  final Map<String, String> buildCacheCredentials = System.getenv("GRADLE_BUILD_CACHE_USERNAME") ? [:] : vault.logical()
+    .read("${vaultPathPrefix}/gradle-build-cache")
     .getData()
   gradle.settingsEvaluated { settings ->
     settings.buildCache {
@@ -104,8 +116,8 @@ if (buildCacheUrl) {
         url = buildCacheUrl
         push = buildCachePush
         credentials {
-          username = buildCacheCredentials.get("username")
-          password = buildCacheCredentials.get("password")
+          username = System.getenv("GRADLE_BUILD_CACHE_USERNAME") ?: buildCacheCredentials.get("username")
+          password = System.getenv("GRADLE_BUILD_CACHE_PASSWORD") ?: buildCacheCredentials.get("password")
         }
       }
     }

.ci/scripts/packaging-test.sh

@@ -39,6 +39,7 @@ if [ -f "/etc/os-release" ] ; then
         # Work around incorrect lintian version
         #  https://github.com/elastic/elasticsearch/issues/48573
         if [ $VERSION_ID == 10 ] ; then
+            sudo apt-get update -y
             sudo apt-get install -y --allow-downgrades lintian=2.15.0
         fi
     fi

.ci/scripts/run-gradle.sh

@@ -1,6 +1,7 @@
 #!/bin/bash
 # drop page cache and kernel slab objects on linux
 [[ -x /usr/local/sbin/drop-caches ]] && sudo /usr/local/sbin/drop-caches
+
 rm -Rfv ~/.gradle/init.d
 mkdir -p ~/.gradle/init.d && cp -v $WORKSPACE/.ci/init.gradle ~/.gradle/init.d
 if [ "$(uname -m)" = "arm64" ] || [ "$(uname -m)" = "aarch64" ]; then

build-tools-internal/src/main/groovy/elasticsearch.build-scan.gradle

@@ -16,7 +16,7 @@ buildScan {
   String buildKiteUrl = System.getenv('BUILDKITE_BUILD_URL') ? System.getenv('BUILDKITE_BUILD_URL') : null
 
   // Automatically publish scans from Elasticsearch CI
-  if (jenkinsUrl?.host?.endsWith('elastic.co') || jenkinsUrl?.host?.endsWith('elastic.dev')) {
+  if (jenkinsUrl?.host?.endsWith('elastic.co') || jenkinsUrl?.host?.endsWith('elastic.dev') || System.getenv('BUILDKITE') == 'true') {
     publishAlways()
     buildScan.server = 'https://gradle-enterprise.elastic.co'
   }

catalog-info.yaml

@@ -16,21 +16,22 @@ spec:
     apiVersion: buildkite.elastic.dev/v1
     kind: Pipeline
     metadata:
-      description: ':elasticsearch: Update elasticsearch submodule in elasticsearch-serverless'
+      description: ":elasticsearch: Update elasticsearch submodule in elasticsearch-serverless"
       name: elasticsearch / update serverless submodule
     spec:
       repository: elastic/elasticsearch
       pipeline_file: .buildkite/update-es-serverless.yml
       teams:
         elasticsearch-team: {}
+        ml-core: {}
         everyone:
           access_level: READ_ONLY
       provider_settings:
         trigger_mode: none
       schedules:
         daily promotion:
           branch: main
-          cronline: '@daily'
+          cronline: "@daily"
 ---
 # yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
 apiVersion: backstage.io/v1alpha1
@@ -49,16 +50,56 @@ spec:
     apiVersion: buildkite.elastic.dev/v1
     kind: Pipeline
     metadata:
-      description: ':elasticsearch: Validate elasticsearch changes against serverless'
+      description: ":elasticsearch: Validate elasticsearch changes against serverless"
       name: elasticsearch / check serverless submodule
     spec:
       repository: elastic/elasticsearch
       pipeline_file: .buildkite/check-es-serverless.yml
       branch_configuration: main
       teams:
         elasticsearch-team: {}
+        ml-core: {}
         everyone:
           access_level: READ_ONLY
       provider_settings:
         build_pull_requests: false
         publish_commit_status: false
+---
+# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/e57ee3bed7a6f73077a3f55a38e76e40ec87a7cf/rre.schema.json
+apiVersion: backstage.io/v1alpha1
+kind: Resource
+metadata:
+  name: buildkite-pipeline-elasticsearch-periodic
+  description: Elasticsearch tests and checks that are run a few times daily
+  links:
+    - title: Pipeline
+      url: https://buildkite.com/elastic/elasticsearch-periodic
+spec:
+  type: buildkite-pipeline
+  system: buildkite
+  owner: group:elasticsearch-team
+  implementation:
+    apiVersion: buildkite.elastic.dev/v1
+    kind: Pipeline
+    metadata:
+      description: ":elasticsearch: Tests and checks that are run a few times daily"
+      name: elasticsearch / periodic
+    spec:
+      repository: elastic/elasticsearch
+      pipeline_file: .buildkite/pipelines/periodic.yml
+      branch_configuration: main
+      teams:
+        elasticsearch-team: {}
+        ml-core: {}
+        everyone:
+          access_level: READ_ONLY
+      provider_settings:
+        build_branches: false
+        build_pull_requests: false
+        publish_commit_status: false
+        trigger_mode: none
+  schedules:
+    Periodically on main:
+      branch: main
+      cronline: "0 0,8,16  * * America/New_York"
+      message: "Tests and checks that are run 3x daily"

distribution/packages/build.gradle

@@ -308,6 +308,7 @@ Closure commonDebConfig(String architecture) {
 
     into('/usr/share/lintian/overrides') {
       from('src/deb/lintian/elasticsearch')
+      fileMode 0644
     }
   }
 }