Merge branch 'lucene_snapshot' into bump_codec_to_Lucene912

.buildkite/pipelines/intake.yml

@@ -62,7 +62,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["7.17.23", "8.14.4", "8.15.0", "8.16.0"]
+            BWC_VERSION: ["7.17.24", "8.14.4", "8.15.0", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -322,8 +322,8 @@ steps:
         env:
           BWC_VERSION: 7.16.3
 
-      - label: "{{matrix.image}} / 7.17.23 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.23
+      - label: "{{matrix.image}} / 7.17.24 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v7.17.24
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -337,7 +337,7 @@ steps:
           buildDirectory: /dev/shm/bk
           diskSizeGb: 250
         env:
-          BWC_VERSION: 7.17.23
+          BWC_VERSION: 7.17.24
 
       - label: "{{matrix.image}} / 8.0.1 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.0.1

.buildkite/pipelines/periodic.yml

@@ -342,8 +342,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 7.17.23 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.23#bwcTest
+      - label: 7.17.24 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v7.17.24#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -353,7 +353,7 @@ steps:
           preemptible: true
           diskSizeGb: 250
         env:
-          BWC_VERSION: 7.17.23
+          BWC_VERSION: 7.17.24
         retry:
           automatic:
             - exit_status: "-1"
@@ -771,7 +771,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk17
-            BWC_VERSION: ["7.17.23", "8.14.4", "8.15.0", "8.16.0"]
+            BWC_VERSION: ["7.17.24", "8.14.4", "8.15.0", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -821,7 +821,7 @@ steps:
               - openjdk21
               - openjdk22
               - openjdk23
-            BWC_VERSION: ["7.17.23", "8.14.4", "8.15.0", "8.16.0"]
+            BWC_VERSION: ["7.17.24", "8.14.4", "8.15.0", "8.16.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.ci/bwcVersions

@@ -16,7 +16,7 @@ BWC_VERSION:
   - "7.14.2"
   - "7.15.2"
   - "7.16.3"
-  - "7.17.23"
+  - "7.17.24"
   - "8.0.1"
   - "8.1.3"
   - "8.2.3"

.ci/snapshotBwcVersions

@@ -1,5 +1,5 @@
 BWC_VERSION:
-  - "7.17.23"
+  - "7.17.24"
   - "8.14.4"
   - "8.15.0"
   - "8.16.0"

README.asciidoc

@@ -33,76 +33,144 @@ https://www.elastic.co/downloads/elasticsearch[elastic.co/downloads/elasticsearc
 === Run Elasticsearch locally
 
 //// 
-IMPORTANT: This content is replicated in the Elasticsearch guide. 
-If you make changes, you must also update setup/set-up-local-dev-deployment.asciidoc.
+IMPORTANT: This content is replicated in the Elasticsearch guide. See `run-elasticsearch-locally.asciidoc`.
+Both will soon be replaced by a quickstart script.
 ////
 
-To try out Elasticsearch on your own machine, we recommend using Docker
-and running both Elasticsearch and Kibana.
-Docker images are available from the https://www.docker.elastic.co[Elastic Docker registry].
+[WARNING]
+==== 
+DO NOT USE THESE INSTRUCTIONS FOR PRODUCTION DEPLOYMENTS.
 
-NOTE: Starting in Elasticsearch 8.0, security is enabled by default. 
-The first time you start Elasticsearch, TLS encryption is configured automatically, 
-a password is generated for the `elastic` user, 
-and a Kibana enrollment token is created so you can connect Kibana to your secured cluster.
+This setup is intended for local development and testing only.
+====
 
-For other installation options, see the
-https://www.elastic.co/guide/en/elasticsearch/reference/current/install-elasticsearch.html[Elasticsearch installation documentation].
+The following commands help you very quickly spin up a single-node Elasticsearch cluster, together with Kibana in Docker.
+Use this setup for local development or testing.
 
-**Start Elasticsearch**
+==== Prerequisites
 
-. Install and start https://www.docker.com/products/docker-desktop[Docker
-Desktop]. Go to **Preferences > Resources > Advanced** and set Memory to at least 4GB.
+If you don't have Docker installed, https://www.docker.com/products/docker-desktop[download and install Docker Desktop] for your operating system.
 
-. Start an Elasticsearch container:
-+
+==== Set environment variables
+
+Configure the following environment variables.
+
+[source,sh]
+----
+export ELASTIC_PASSWORD="<ES_PASSWORD>"  # password for "elastic" username
+export KIBANA_PASSWORD="<KIB_PASSWORD>"   # Used internally by Kibana, must be at least 6 characters long
+----
+
+==== Create a Docker network
+
+To run both Elasticsearch and Kibana, you'll need to create a Docker network:
+
+[source,sh]
 ----
-docker network create elastic
-docker pull docker.elastic.co/elasticsearch/elasticsearch:{version} <1>
-docker run --name elasticsearch --net elastic -p 9200:9200 -p 9300:9300 -e "discovery.type=single-node" -t docker.elastic.co/elasticsearch/elasticsearch:{version}
+docker network create elastic-net
 ----
-<1> Replace {version} with the version of Elasticsearch you want to run.
-+
-When you start Elasticsearch for the first time, the generated `elastic` user password and
-Kibana enrollment token are output to the terminal.
-+
-NOTE: You might need to scroll back a bit in the terminal to view the password 
-and enrollment token.
 
-. Copy the generated password and enrollment token and save them in a secure 
-location. These values are shown only when you start Elasticsearch for the first time.
-You'll use these to enroll Kibana with your Elasticsearch cluster and log in.
+==== Run Elasticsearch
+
+Start the Elasticsearch container with the following command:
 
-**Start Kibana**
+[source,sh]
+----
+docker run -p 127.0.0.1:9200:9200 -d --name elasticsearch --network elastic-net \
+  -e ELASTIC_PASSWORD=$ELASTIC_PASSWORD \
+  -e "discovery.type=single-node" \
+  -e "xpack.security.http.ssl.enabled=false" \
+  -e "xpack.license.self_generated.type=trial" \
+  docker.elastic.co/elasticsearch/elasticsearch:{version}
+----
 
-Kibana enables you to easily send requests to Elasticsearch and analyze, visualize, and manage data interactively.
+==== Run Kibana (optional)
 
-. In a new terminal session, start Kibana and connect it to your Elasticsearch container:
-+
+To run Kibana, you must first set the `kibana_system` password in the Elasticsearch container.
+
+[source,sh]
 ----
-docker pull docker.elastic.co/kibana/kibana:{version} <1>
-docker run --name kibana --net elastic -p 5601:5601 docker.elastic.co/kibana/kibana:{version}
+# configure the Kibana password in the ES container
+curl -u elastic:$ELASTIC_PASSWORD \
+  -X POST \
+  http://localhost:9200/_security/user/kibana_system/_password \
+  -d '{"password":"'"$KIBANA_PASSWORD"'"}' \
+  -H 'Content-Type: application/json'
 ----
-<1> Replace {version} with the version of Kibana you want to run.
-+
-When you start Kibana, a unique URL is output to your terminal.
+// NOTCONSOLE
 
-. To access Kibana, open the generated URL in your browser.
+Start the Kibana container with the following command:
 
-  .. Paste the enrollment token that you copied when starting
-  Elasticsearch and click the button to connect your Kibana instance with Elasticsearch.
+[source,sh]
+----
+docker run -p 127.0.0.1:5601:5601 -d --name kibana --network elastic-net \
+  -e ELASTICSEARCH_URL=http://elasticsearch:9200 \
+  -e ELASTICSEARCH_HOSTS=http://elasticsearch:9200 \
+  -e ELASTICSEARCH_USERNAME=kibana_system \
+  -e ELASTICSEARCH_PASSWORD=$KIBANA_PASSWORD \
+  -e "xpack.security.enabled=false" \
+  -e "xpack.license.self_generated.type=trial" \
+  docker.elastic.co/kibana/kibana:{version}
+----
 
-  .. Log in to Kibana as the `elastic` user with the password that was generated
-  when you started Elasticsearch.
+.Trial license
+[%collapsible]
+====
+The service is started with a trial license. The trial license enables all features of Elasticsearch for a trial period of 30 days. After the trial period expires, the license is downgraded to a basic license, which is free forever. If you prefer to skip the trial and use the basic license, set the value of the `xpack.license.self_generated.type` variable to basic instead. For a detailed feature comparison between the different licenses, refer to our https://www.elastic.co/subscriptions[subscriptions page].
+====
 
-**Send requests to Elasticsearch**
+==== Send requests to Elasticsearch
 
 You send data and other requests to Elasticsearch through REST APIs. 
 You can interact with Elasticsearch using any client that sends HTTP requests, 
 such as the https://www.elastic.co/guide/en/elasticsearch/client/index.html[Elasticsearch
 language clients] and https://curl.se[curl]. 
+
+===== Using curl 
+
+Here's an example curl command to create a new Elasticsearch index, using basic auth:
+
+[source,sh]
+----
+curl -u elastic:$ELASTIC_PASSWORD \
+  -X PUT \
+  http://localhost:9200/my-new-index \
+  -H 'Content-Type: application/json'
+----
+// NOTCONSOLE
+
+===== Using a language client
+
+To connect to your local dev Elasticsearch cluster with a language client, you can use basic authentication with the `elastic` username and the password you set in the environment variable.
+
+You'll use the following connection details:
+
+* **Elasticsearch endpoint**: `http://localhost:9200`
+* **Username**: `elastic`
+* **Password**: `$ELASTIC_PASSWORD` (Value you set in the environment variable)
+
+For example, to connect with the Python `elasticsearch` client:
+
+[source,python]
+----
+import os
+from elasticsearch import Elasticsearch
+
+username = 'elastic'
+password = os.getenv('ELASTIC_PASSWORD') # Value you set in the environment variable
+
+client = Elasticsearch(
+    "http://localhost:9200",
+    basic_auth=(username, password)
+)
+
+print(client.info())
+----
+
+===== Using the Dev Tools Console
+
 Kibana's developer console provides an easy way to experiment and test requests. 
-To access the console, go to **Management > Dev Tools**.
+To access the console, open Kibana, then go to **Management** > **Dev Tools**.
 
 **Add data**
 

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/ReleaseToolsPlugin.java

@@ -52,7 +52,7 @@ public void apply(Project project) {
 
         project.getTasks().register("extractCurrentVersions", ExtractCurrentVersionsTask.class);
         project.getTasks().register("tagVersions", TagVersionsTask.class);
-        project.getTasks().register("setCompatibleVersions", SetCompatibleVersionsTask.class);
+        project.getTasks().register("setCompatibleVersions", SetCompatibleVersionsTask.class, t -> t.setThisVersion(version));
 
         final FileTree yamlFiles = projectDirectory.dir("docs/changelog")
             .getAsFileTree()

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/release/SetCompatibleVersionsTask.java

@@ -14,6 +14,7 @@
 import com.github.javaparser.ast.expr.NameExpr;
 import com.github.javaparser.printer.lexicalpreservation.LexicalPreservingPrinter;
 
+import org.elasticsearch.gradle.Version;
 import org.gradle.api.tasks.TaskAction;
 import org.gradle.api.tasks.options.Option;
 import org.gradle.initialization.layout.BuildLayout;
@@ -28,28 +29,44 @@
 
 public class SetCompatibleVersionsTask extends AbstractVersionsTask {
 
+    private Version thisVersion;
+    private Version releaseVersion;
     private Map<String, Integer> versionIds = Map.of();
 
     @Inject
     public SetCompatibleVersionsTask(BuildLayout layout) {
         super(layout);
     }
 
+    public void setThisVersion(Version version) {
+        thisVersion = version;
+    }
+
     @Option(option = "version-id", description = "Version id used for the release. Of the form <VersionType>:<id>.")
     public void versionIds(List<String> version) {
         this.versionIds = splitVersionIds(version);
     }
 
+    @Option(option = "release", description = "The version being released")
+    public void releaseVersion(String version) {
+        releaseVersion = Version.fromString(version);
+    }
+
     @TaskAction
     public void executeTask() throws IOException {
         if (versionIds.isEmpty()) {
             throw new IllegalArgumentException("No version ids specified");
         }
+
+        if (releaseVersion.getMajor() < thisVersion.getMajor()) {
+            // don't need to update CCS version - this is for a different major
+            return;
+        }
+
         Integer transportVersion = versionIds.get(TRANSPORT_VERSION_TYPE);
         if (transportVersion == null) {
             throw new IllegalArgumentException("TransportVersion id not specified");
         }
-
         Path versionJava = rootDir.resolve(TRANSPORT_VERSIONS_FILE_PATH);
         CompilationUnit file = LexicalPreservingPrinter.setup(StaticJavaParser.parse(versionJava));
 

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 8.16.0
-lucene            = 9.12.0-snapshot-e83c1d4234c
+lucene            = 9.12.0-snapshot-65be22a6221
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 22.0.1+8@c7ec1332f7bb44aeba2eb341ae18aca4

docs/changelog/110216.yaml

@@ -0,0 +1,5 @@
+pr: 110216
+summary: Register SLM run before snapshotting to save stats
+area: ILM+SLM
+type: enhancement
+issues: []

docs/changelog/111245.yaml

@@ -0,0 +1,6 @@
+pr: 111245
+summary: Truncating watcher history if it is too large
+area: Watcher
+type: bug
+issues:
+ - 94745

docs/changelog/111369.yaml

@@ -0,0 +1,5 @@
+pr: 111369
+summary: Improve security-crypto threadpool overflow handling
+area: Authentication
+type: bug
+issues: []

docs/changelog/111420.yaml

@@ -0,0 +1,5 @@
+pr: 111420
+summary: "[Query rules] Add `exclude` query rule type"
+area: Relevance
+type: feature
+issues: []

docs/changelog/111519.yaml

@@ -0,0 +1,5 @@
+pr: 111519
+summary: "ESQL: Don't mutate the `BoolQueryBuilder` in plan"
+area: ES|QL
+type: bug
+issues: []