Merge remote-tracking branch 'origin/main' into lucene_snapshot

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/VectorScorerBenchmark.java

@@ -83,6 +83,8 @@ public class VectorScorerBenchmark {
 
     RandomVectorScorer luceneDotScorerQuery;
     RandomVectorScorer nativeDotScorerQuery;
+    RandomVectorScorer luceneSqrScorerQuery;
+    RandomVectorScorer nativeSqrScorerQuery;
 
     @Setup
     public void setup() throws IOException {
@@ -130,6 +132,8 @@ public void setup() throws IOException {
         }
         luceneDotScorerQuery = luceneScorer(values, VectorSimilarityFunction.DOT_PRODUCT, queryVec);
         nativeDotScorerQuery = factory.getInt7SQVectorScorer(VectorSimilarityFunction.DOT_PRODUCT, values, queryVec).get();
+        luceneSqrScorerQuery = luceneScorer(values, VectorSimilarityFunction.EUCLIDEAN, queryVec);
+        nativeSqrScorerQuery = factory.getInt7SQVectorScorer(VectorSimilarityFunction.EUCLIDEAN, values, queryVec).get();
 
         // sanity
         var f1 = dotProductLucene();
@@ -157,6 +161,12 @@ public void setup() throws IOException {
         if (q1 != q2) {
             throw new AssertionError("query: lucene[" + q1 + "] != " + "native[" + q2 + "]");
         }
+
+        var sqr1 = squareDistanceLuceneQuery();
+        var sqr2 = squareDistanceNativeQuery();
+        if (sqr1 != sqr2) {
+            throw new AssertionError("query: lucene[" + q1 + "] != " + "native[" + q2 + "]");
+        }
     }
 
     @TearDown
@@ -217,6 +227,16 @@ public float squareDistanceScalar() {
         return 1 / (1f + adjustedDistance);
     }
 
+    @Benchmark
+    public float squareDistanceLuceneQuery() throws IOException {
+        return luceneSqrScorerQuery.score(1);
+    }
+
+    @Benchmark
+    public float squareDistanceNativeQuery() throws IOException {
+        return nativeSqrScorerQuery.score(1);
+    }
+
     QuantizedByteVectorValues vectorValues(int dims, int size, IndexInput in, VectorSimilarityFunction sim) throws IOException {
         var sq = new ScalarQuantizer(0.1f, 0.9f, (byte) 7);
         var slice = in.slice("values", 0, in.length());

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/GitInfoPlugin.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.gradle.internal.conventions;
 
 import org.elasticsearch.gradle.internal.conventions.info.GitInfo;
+import org.elasticsearch.gradle.internal.conventions.info.GitInfoValueSource;
 import org.elasticsearch.gradle.internal.conventions.util.Util;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
@@ -18,38 +19,28 @@
 import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
 
-import javax.inject.Inject;
 import java.io.File;
 
-class GitInfoPlugin implements Plugin<Project> {
+import javax.inject.Inject;
 
-    private ProviderFactory factory;
-    private ObjectFactory objectFactory;
+public abstract class GitInfoPlugin implements Plugin<Project> {
 
+    private ProviderFactory factory;
     private Provider<String> revision;
-    private Property<GitInfo> gitInfo;
 
     @Inject
-    GitInfoPlugin(ProviderFactory factory, ObjectFactory objectFactory) {
+    public GitInfoPlugin(ProviderFactory factory) {
         this.factory = factory;
-        this.objectFactory = objectFactory;
     }
 
     @Override
     public void apply(Project project) {
         File rootDir = Util.locateElasticsearchWorkspace(project.getGradle());
-        gitInfo = objectFactory.property(GitInfo.class).value(factory.provider(() ->
-            GitInfo.gitInfo(rootDir)
-        ));
-        gitInfo.disallowChanges();
-        gitInfo.finalizeValueOnRead();
-
-        revision = gitInfo.map(info -> info.getRevision() == null ? info.getRevision() : "main");
+        getGitInfo().convention(factory.of(GitInfoValueSource.class, spec -> { spec.getParameters().getPath().set(rootDir); }));
+        revision = getGitInfo().map(info -> info.getRevision() == null ? info.getRevision() : "main");
     }
 
-    public Property<GitInfo> getGitInfo() {
-        return gitInfo;
-    }
+    public abstract Property<GitInfo> getGitInfo();
 
     public Provider<String> getRevision() {
         return revision;

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/LicensingPlugin.java

@@ -15,9 +15,10 @@
 import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
 
-import javax.inject.Inject;
 import java.util.Map;
 
+import javax.inject.Inject;
+
 public class LicensingPlugin implements Plugin<Project> {
     static final String ELASTIC_LICENSE_URL_PREFIX = "https://raw.githubusercontent.com/elastic/elasticsearch/";
     static final String ELASTIC_LICENSE_URL_POSTFIX = "/licenses/ELASTIC-LICENSE-2.0.txt";
@@ -33,24 +34,33 @@ public LicensingPlugin(ProviderFactory providerFactory) {
     @Override
     public void apply(Project project) {
         Provider<String> revision = project.getRootProject().getPlugins().apply(GitInfoPlugin.class).getRevision();
-        Provider<String> licenseCommitProvider = providerFactory.provider(() ->
-             isSnapshotVersion(project) ? revision.get() : "v" + project.getVersion()
+        Provider<String> licenseCommitProvider = providerFactory.provider(
+            () -> isSnapshotVersion(project) ? revision.get() : "v" + project.getVersion()
         );
 
-        Provider<String> elasticLicenseURL = licenseCommitProvider.map(licenseCommit -> ELASTIC_LICENSE_URL_PREFIX +
-                licenseCommit + ELASTIC_LICENSE_URL_POSTFIX);
-        Provider<String> agplLicenseURL = licenseCommitProvider.map(licenseCommit -> ELASTIC_LICENSE_URL_PREFIX +
-            licenseCommit + AGPL_ELASTIC_LICENSE_URL_POSTFIX);
+        Provider<String> elasticLicenseURL = licenseCommitProvider.map(
+            licenseCommit -> ELASTIC_LICENSE_URL_PREFIX + licenseCommit + ELASTIC_LICENSE_URL_POSTFIX
+        );
+        Provider<String> agplLicenseURL = licenseCommitProvider.map(
+            licenseCommit -> ELASTIC_LICENSE_URL_PREFIX + licenseCommit + AGPL_ELASTIC_LICENSE_URL_POSTFIX
+        );
         // But stick the Elastic license url in project.ext so we can get it if we need to switch to it
         project.getExtensions().getExtraProperties().set("elasticLicenseUrl", elasticLicenseURL);
 
-        MapProperty<String, String> licensesProperty = project.getObjects().mapProperty(String.class, String.class).convention(
-                providerFactory.provider(() -> Map.of(
-                        "Server Side Public License, v 1", "https://www.mongodb.com/licensing/server-side-public-license",
-                        "Elastic License 2.0", elasticLicenseURL.get(),
-                        "GNU Affero General Public License Version 3", agplLicenseURL.get())
+        MapProperty<String, Provider<String>> licensesProperty = project.getObjects()
+            .mapProperty(String.class, (Class<Provider<String>>) (Class<?>) Provider.class)
+            .convention(
+                providerFactory.provider(
+                    () -> Map.of(
+                        "Server Side Public License, v 1",
+                        providerFactory.provider(() -> "https://www.mongodb.com/licensing/server-side-public-license"),
+                        "Elastic License 2.0",
+                        elasticLicenseURL,
+                        "GNU Affero General Public License Version 3",
+                        agplLicenseURL
+                    )
                 )
-        );
+            );
 
         // Default to the SSPL+Elastic dual license
         project.getExtensions().getExtraProperties().set("projectLicenses", licensesProperty);

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/PublishPlugin.java

@@ -28,6 +28,7 @@
 import org.gradle.api.plugins.JavaLibraryPlugin;
 import org.gradle.api.plugins.JavaPlugin;
 import org.gradle.api.provider.MapProperty;
+import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.api.publish.PublishingExtension;
 import org.gradle.api.publish.maven.MavenPublication;
@@ -42,6 +43,7 @@
 import java.io.File;
 import java.util.Map;
 import java.util.concurrent.Callable;
+
 import javax.inject.Inject;
 
 public class PublishPlugin implements Plugin<Project> {
@@ -81,15 +83,15 @@ private void configurePublications(Project project) {
             }
         });
         @SuppressWarnings("unchecked")
-        var projectLicenses = (MapProperty<String, String>) project.getExtensions().getExtraProperties().get("projectLicenses");
+        var projectLicenses = (MapProperty<String, Provider<String>>) project.getExtensions().getExtraProperties().get("projectLicenses");
         publication.getPom().withXml(xml -> {
             var node = xml.asNode();
             node.appendNode("inceptionYear", "2009");
             var licensesNode = node.appendNode("licenses");
             projectLicenses.get().entrySet().stream().sorted(Map.Entry.comparingByKey()).forEach(entry -> {
                 Node license = licensesNode.appendNode("license");
                 license.appendNode("name", entry.getKey());
-                license.appendNode("url", entry.getValue());
+                license.appendNode("url", entry.getValue().get());
                 license.appendNode("distribution", "repo");
             });
             var developer = node.appendNode("developers").appendNode("developer");
@@ -194,7 +196,6 @@ static void configureSourcesJar(Project project) {
         });
     }
 
-
     /**
      * Format the generated pom files to be in a sort of reproducible order.
      */

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/GitInfo.java

@@ -22,6 +22,7 @@
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
+import java.util.Objects;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.stream.Collectors;
@@ -190,4 +191,15 @@ public String urlFromOrigin() {
         }
     }
 
+    @Override
+    public boolean equals(Object o) {
+        if (o == null || getClass() != o.getClass()) return false;
+        GitInfo gitInfo = (GitInfo) o;
+        return Objects.equals(revision, gitInfo.revision) && Objects.equals(origin, gitInfo.origin);
+    }
+
+    @Override
+    public int hashCode() {
+        return Objects.hash(revision, origin);
+    }
 }

build-conventions/src/main/java/org/elasticsearch/gradle/internal/conventions/info/GitInfoValueSource.java

@@ -0,0 +1,22 @@
+package org.elasticsearch.gradle.internal.conventions.info;
+
+import org.gradle.api.provider.Property;
+import org.gradle.api.provider.ValueSource;
+import org.gradle.api.provider.ValueSourceParameters;
+import org.jetbrains.annotations.Nullable;
+
+import java.io.File;
+
+public abstract class GitInfoValueSource implements ValueSource<GitInfo, GitInfoValueSource.Parameters> {
+
+    @Nullable
+    @Override
+    public GitInfo obtain() {
+        File path = getParameters().getPath().get();
+        return GitInfo.gitInfo(path);
+    }
+
+    public interface Parameters extends ValueSourceParameters {
+        Property<File> getPath();
+    }
+}

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/PublishPluginFuncTest.groovy

@@ -45,7 +45,8 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
         file("build/distributions/hello-world-1.0-javadoc.jar").exists()
         file("build/distributions/hello-world-1.0-sources.jar").exists()
         file("build/distributions/hello-world-1.0.pom").exists()
-        assertXmlEquals(file("build/distributions/hello-world-1.0.pom").text, """
+        assertXmlEquals(
+            file("build/distributions/hello-world-1.0.pom").text, """
             <project xmlns="http://maven.apache.org/POM/4.0.0" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
   <!-- This module was also published with a richer model, Gradle metadata,  -->
   <!-- which should be used instead. Do not delete the following line which  -->
@@ -130,7 +131,8 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
         file("build/distributions/hello-world-1.0-javadoc.jar").exists()
         file("build/distributions/hello-world-1.0-sources.jar").exists()
         file("build/distributions/hello-world-1.0.pom").exists()
-        assertXmlEquals(file("build/distributions/hello-world-1.0.pom").text, """
+        assertXmlEquals(
+            file("build/distributions/hello-world-1.0.pom").text, """
             <project xmlns="http://maven.apache.org/POM/4.0.0" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
               <modelVersion>4.0.0</modelVersion>
               <groupId>org.acme</groupId>
@@ -219,7 +221,8 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
         file("build/distributions/hello-world-1.0-javadoc.jar").exists()
         file("build/distributions/hello-world-1.0-sources.jar").exists()
         file("build/distributions/hello-world-1.0.pom").exists()
-        assertXmlEquals(file("build/distributions/hello-world-1.0.pom").text, """
+        assertXmlEquals(
+            file("build/distributions/hello-world-1.0.pom").text, """
             <project xmlns="http://maven.apache.org/POM/4.0.0" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
               <modelVersion>4.0.0</modelVersion>
               <groupId>org.acme</groupId>
@@ -312,7 +315,8 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
         file("build/distributions/hello-world-plugin-1.0-javadoc.jar").exists()
         file("build/distributions/hello-world-plugin-1.0-sources.jar").exists()
         file("build/distributions/hello-world-plugin-1.0.pom").exists()
-        assertXmlEquals(file("build/distributions/hello-world-plugin-1.0.pom").text, """
+        assertXmlEquals(
+            file("build/distributions/hello-world-plugin-1.0.pom").text, """
             <project xmlns="http://maven.apache.org/POM/4.0.0" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
               <!-- This module was also published with a richer model, Gradle metadata,  -->
               <!-- which should be used instead. Do not delete the following line which  -->
@@ -389,7 +393,8 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
         then:
         result.task(":generatePom").outcome == TaskOutcome.SUCCESS
         file("build/distributions/hello-world-plugin-2.0.pom").exists()
-        assertXmlEquals(file("build/distributions/hello-world-plugin-2.0.pom").text, """
+        assertXmlEquals(
+            file("build/distributions/hello-world-plugin-2.0.pom").text, """
             <project xmlns="http://maven.apache.org/POM/4.0.0" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
               <!-- This module was also published with a richer model, Gradle metadata,  -->
               <!-- which should be used instead. Do not delete the following line which  -->
@@ -439,15 +444,15 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
         // scm info only added for internal builds
         internalBuild()
         buildFile << """
-            buildParams.setGitOrigin("https://some-repo.com/repo.git")
+            buildParams.setGitOrigin(project.providers.provider(() -> "https://some-repo.com/repo.git"))
             apply plugin:'elasticsearch.java'
             apply plugin:'elasticsearch.publish'
 
             version = "1.0"
             group = 'org.acme'
             description = "just a test project"
 
-            ext.projectLicenses.set(['The Apache Software License, Version 2.0': 'http://www.apache.org/licenses/LICENSE-2.0'])
+            ext.projectLicenses.set(['The Apache Software License, Version 2.0': project.providers.provider(() -> 'http://www.apache.org/licenses/LICENSE-2.0')])
         """
 
         when:
@@ -456,7 +461,8 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
         then:
         result.task(":generatePom").outcome == TaskOutcome.SUCCESS
         file("build/distributions/hello-world-1.0.pom").exists()
-        assertXmlEquals(file("build/distributions/hello-world-1.0.pom").text, """
+        assertXmlEquals(
+            file("build/distributions/hello-world-1.0.pom").text, """
             <project xmlns="http://maven.apache.org/POM/4.0.0" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
           <!-- This module was also published with a richer model, Gradle metadata,  -->
           <!-- which should be used instead. Do not delete the following line which  -->
@@ -493,15 +499,15 @@ class PublishPluginFuncTest extends AbstractGradleFuncTest {
 
     private boolean assertXmlEquals(String toTest, String expected) {
         def diff = DiffBuilder.compare(Input.fromString(expected))
-                .ignoreWhitespace()
-                .ignoreComments()
-                .normalizeWhitespace()
-                .withTest(Input.fromString(toTest))
-                .build()
+            .ignoreWhitespace()
+            .ignoreComments()
+            .normalizeWhitespace()
+            .withTest(Input.fromString(toTest))
+            .build()
         diff.differences.each { difference ->
             println difference
         }
-        if(diff.differences.size() > 0) {
+        if (diff.differences.size() > 0) {
             println """ given:
 $toTest
 """

build-tools-internal/src/main/groovy/elasticsearch.ide.gradle

@@ -166,7 +166,7 @@ if (providers.systemProperty('idea.active').getOrNull() == 'true') {
   tasks.register('buildDependencyArtifacts') {
     group = 'ide'
     description = 'Builds artifacts needed as dependency for IDE modules'
-    dependsOn([':plugins:repository-hdfs:hadoop-client-api:shadowJar',
+    dependsOn([':plugins:repository-hdfs:hadoop-client-api:jar',
       ':x-pack:plugin:esql:compute:ann:jar',
       ':x-pack:plugin:esql:compute:gen:jar',
       ':server:generateModulesList',

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaPlugin.java

@@ -27,6 +27,7 @@
 import org.gradle.api.plugins.JavaLibraryPlugin;
 import org.gradle.api.plugins.JavaPlugin;
 import org.gradle.api.provider.Property;
+import org.gradle.api.provider.Provider;
 import org.gradle.api.tasks.TaskProvider;
 import org.gradle.api.tasks.bundling.Jar;
 import org.gradle.api.tasks.javadoc.Javadoc;
@@ -120,12 +121,12 @@ public void execute(Task task) {
     }
 
     private static void configureJarManifest(Project project, BuildParameterExtension buildParams) {
-        String gitOrigin = buildParams.getGitOrigin();
-        String gitRevision = buildParams.getGitRevision();
+        Provider<String> gitOrigin = buildParams.getGitOrigin();
+        Provider<String> gitRevision = buildParams.getGitRevision();
 
         project.getPlugins().withType(InfoBrokerPlugin.class).whenPluginAdded(manifestPlugin -> {
-            manifestPlugin.add("Module-Origin", toStringable(() -> gitOrigin));
-            manifestPlugin.add("Change", toStringable(() -> gitRevision));
+            manifestPlugin.add("Module-Origin", toStringable(() -> gitOrigin.get()));
+            manifestPlugin.add("Change", toStringable(() -> gitRevision.get()));
             manifestPlugin.add("X-Compile-Elasticsearch-Version", toStringable(VersionProperties::getElasticsearch));
             manifestPlugin.add("X-Compile-Lucene-Version", toStringable(VersionProperties::getLucene));
             manifestPlugin.add(

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/info/BuildParameterExtension.java

@@ -46,9 +46,9 @@ public interface BuildParameterExtension {
 
     Provider<String> getRuntimeJavaDetails();
 
-    String getGitRevision();
+    Provider<String> getGitRevision();
 
-    String getGitOrigin();
+    Provider<String> getGitOrigin();
 
     ZonedDateTime getBuildDate();