Merge branch 'main' into lazy-failure-store-creation

# Conflicts: # server/src/main/java/org/elasticsearch/TransportVersions.java
nielsbauman · Jun 4, 2024 · b5c25d4 · b5c25d4
2 parents e257a90 + be502cc
commit b5c25d4
Show file tree

Hide file tree

Showing 368 changed files with 7,204 additions and 1,484 deletions.
diff --git a/build-conventions/build.gradle b/build-conventions/build.gradle
@@ -8,6 +8,16 @@
 
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
 
+
+buildscript {
+  repositories {
+    maven {
+      url 'https://jitpack.io'
+    }
+    mavenCentral()
+  }
+}
+
 plugins {
     id 'java-gradle-plugin'
     id 'java-test-fixtures'
@@ -59,6 +69,10 @@ gradlePlugin {
 }
 
 repositories {
+    maven {
+      url 'https://jitpack.io'
+    }
+
     mavenCentral()
     gradlePluginPortal()
 }

diff --git a/build-tools-internal/build.gradle b/build-tools-internal/build.gradle
@@ -257,6 +257,9 @@ tasks.named('licenseHeaders').configure {
  *****************************************************************************/
 
 repositories {
+  maven {
+    url 'https://jitpack.io'
+  }
   mavenCentral()
   gradlePluginPortal()
 }

diff --git a/build-tools-internal/gradle/wrapper/gradle-wrapper.properties b/build-tools-internal/gradle/wrapper/gradle-wrapper.properties
@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=194717442575a6f96e1c1befa2c30e9a4fc90f701d7aee33eb879b79e7ff05c0
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.7-all.zip
+distributionSha256Sum=f8b4f4772d302c8ff580bc40d0f56e715de69b163546944f787c87abf209c961
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

diff --git a/build-tools-internal/settings.gradle b/build-tools-internal/settings.gradle
@@ -1,5 +1,13 @@
 pluginManagement {
-    includeBuild "../build-conventions"
+  repositories {
+    maven {
+      url 'https://jitpack.io'
+    }
+    mavenCentral()
+    gradlePluginPortal()
+  }
+
+  includeBuild "../build-conventions"
     includeBuild "../build-tools"
 }
 
@@ -9,4 +17,4 @@ dependencyResolutionManagement {
             from(files("../gradle/build.versions.toml"))
         }
     }
-}
+}
diff --git a/...c/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java b/...c/main/java/org/elasticsearch/gradle/internal/InternalDistributionArchiveSetupPlugin.java
@@ -99,8 +99,8 @@ private void configureGeneralTaskDefaults(Project project) {
         project.getTasks().withType(AbstractCopyTask.class).configureEach(t -> {
             t.dependsOn(project.getTasks().withType(EmptyDirTask.class));
             t.setIncludeEmptyDirs(true);
-            t.setDirMode(0755);
-            t.setFileMode(0644);
+            t.dirPermissions(permissions -> permissions.unix(0755));
+            t.filePermissions(permissions -> permissions.unix(0644));
         });
 
         // common config across all archives

diff --git a/...s-internal/src/main/java/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTar.java b/...s-internal/src/main/java/org/elasticsearch/gradle/internal/SymbolicLinkPreservingTar.java
@@ -145,7 +145,7 @@ private void visitSymbolicLink(final FileCopyDetailsInternal details) {
                 visitedSymbolicLinks.add(details.getFile());
                 final TarArchiveEntry entry = new TarArchiveEntry(details.getRelativePath().getPathString(), TarConstants.LF_SYMLINK);
                 entry.setModTime(getModTime(details));
-                entry.setMode(UnixStat.LINK_FLAG | details.getMode());
+                entry.setMode(UnixStat.LINK_FLAG | details.getPermissions().toUnixNumeric());
                 try {
                     entry.setLinkName(Files.readSymbolicLink(details.getFile().toPath()).toString());
                     tar.putArchiveEntry(entry);
@@ -158,7 +158,7 @@ private void visitSymbolicLink(final FileCopyDetailsInternal details) {
             private void visitDirectory(final FileCopyDetailsInternal details) {
                 final TarArchiveEntry entry = new TarArchiveEntry(details.getRelativePath().getPathString() + "/");
                 entry.setModTime(getModTime(details));
-                entry.setMode(UnixStat.DIR_FLAG | details.getMode());
+                entry.setMode(UnixStat.DIR_FLAG | details.getPermissions().toUnixNumeric());
                 try {
                     tar.putArchiveEntry(entry);
                     tar.closeArchiveEntry();
@@ -170,7 +170,7 @@ private void visitDirectory(final FileCopyDetailsInternal details) {
             private void visitFile(final FileCopyDetailsInternal details) {
                 final TarArchiveEntry entry = new TarArchiveEntry(details.getRelativePath().getPathString());
                 entry.setModTime(getModTime(details));
-                entry.setMode(UnixStat.FILE_FLAG | details.getMode());
+                entry.setMode(UnixStat.FILE_FLAG | details.getPermissions().toUnixNumeric());
                 entry.setSize(details.getSize());
                 try {
                     tar.putArchiveEntry(entry);

diff --git a/...-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java b/...-internal/src/main/java/org/elasticsearch/gradle/internal/info/GlobalBuildInfoPlugin.java
@@ -260,7 +260,7 @@ private List<JavaHome> getAvailableJavaVersions() {
     private Stream<InstallationLocation> getAvailableJavaInstallationLocationSteam() {
         return Stream.concat(
             javaInstallationRegistry.toolchains().stream().map(metadata -> metadata.location),
-            Stream.of(new InstallationLocation(Jvm.current().getJavaHome(), "Current JVM"))
+            Stream.of(InstallationLocation.userDefined(Jvm.current().getJavaHome(), "Current JVM"))
         );
     }
 

diff --git a/...n/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesPrecommitPlugin.java b/...n/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesPrecommitPlugin.java
@@ -12,30 +12,23 @@
 import org.elasticsearch.gradle.internal.conventions.precommit.PrecommitPlugin;
 import org.gradle.api.Project;
 import org.gradle.api.Task;
-import org.gradle.api.artifacts.Configuration;
-import org.gradle.api.artifacts.ProjectDependency;
+import org.gradle.api.artifacts.component.ComponentIdentifier;
+import org.gradle.api.artifacts.component.ModuleComponentIdentifier;
 import org.gradle.api.plugins.JavaPlugin;
+import org.gradle.api.specs.Spec;
 import org.gradle.api.tasks.TaskProvider;
 
 public class DependencyLicensesPrecommitPlugin extends PrecommitPlugin {
+    private static Spec<ComponentIdentifier> COMPONENT_FILTER = identifier -> (identifier instanceof ModuleComponentIdentifier)
+        && ((ModuleComponentIdentifier) identifier).getGroup().startsWith("org.elasticsearch") == false;
 
     @Override
     public TaskProvider<? extends Task> createTask(Project project) {
         project.getPlugins().apply(CompileOnlyResolvePlugin.class);
-        TaskProvider<DependencyLicensesTask> dependencyLicenses = project.getTasks()
-            .register("dependencyLicenses", DependencyLicensesTask.class);
-
-        // only require dependency licenses for non-elasticsearch deps
-        dependencyLicenses.configure(t -> {
-            Configuration runtimeClasspath = project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME);
-            Configuration compileOnly = project.getConfigurations()
-                .getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
-            t.setDependencies(
-                runtimeClasspath.fileCollection(
-                    dependency -> dependency instanceof ProjectDependency == false
-                        && dependency.getGroup().startsWith("org.elasticsearch") == false
-                ).minus(compileOnly)
-            );
+        var dependencyLicenses = project.getTasks().register("dependencyLicenses", DependencyLicensesTask.class, t -> {
+            var runtimeClasspath = project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME);
+            var compileOnly = project.getConfigurations().getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
+            t.configureDependencies(runtimeClasspath, compileOnly, COMPONENT_FILTER);
         });
         return dependencyLicenses;
     }

diff --git a/...nal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java b/...nal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java
@@ -11,14 +11,18 @@
 import org.gradle.api.DefaultTask;
 import org.gradle.api.GradleException;
 import org.gradle.api.InvalidUserDataException;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.component.ComponentIdentifier;
 import org.gradle.api.file.Directory;
 import org.gradle.api.file.DirectoryProperty;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.file.ProjectLayout;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
 import org.gradle.api.model.ObjectFactory;
+import org.gradle.api.provider.Property;
 import org.gradle.api.provider.Provider;
+import org.gradle.api.specs.Spec;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputDirectory;
 import org.gradle.api.tasks.InputFiles;
@@ -41,6 +45,8 @@
 
 import javax.inject.Inject;
 
+import static org.elasticsearch.gradle.internal.util.DependenciesUtils.createFileCollectionFromNonTransitiveArtifactsView;
+
 /**
  * A task to check licenses for dependencies.
  * <p>
@@ -83,7 +89,7 @@
  * for the dependency. This artifact will be redistributed by us with the release to
  * comply with the license terms.
  */
-public class DependencyLicensesTask extends DefaultTask {
+public abstract class DependencyLicensesTask extends DefaultTask {
 
     private final Pattern regex = Pattern.compile("-v?\\d+.*");
 
@@ -181,6 +187,10 @@ public void ignoreFile(String file) {
         ignoreFiles.add(file);
     }
 
+    @Input
+    @Optional
+    public abstract Property<Spec<ComponentIdentifier>> getComponentFilter();
+
     @TaskAction
     public void checkDependencies() {
         if (dependencies == null) {
@@ -295,7 +305,6 @@ private String getFileName(String name, Map<String, ?> counters, String type) {
             // try the other suffix...TODO: get rid of this, just support ending in .txt
             return fileName + ".txt";
         }
-
         return fileName;
     }
 
@@ -310,4 +319,15 @@ public LinkedHashMap<String, String> getMappings() {
         return new LinkedHashMap<>(mappings);
     }
 
+    /**
+     * Convencience method for configuring dependencies to be checked and ignoring transitive dependencies for now.
+     * */
+    public void configureDependencies(
+        Configuration plusConfiguration,
+        Configuration minusConfiguration,
+        Spec<ComponentIdentifier> componentFilter
+    ) {
+        setDependencies(createFileCollectionFromNonTransitiveArtifactsView(plusConfiguration, componentFilter).minus(minusConfiguration));
+    }
+
 }
diff --git a/...main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java b/...main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java
@@ -15,11 +15,14 @@
 import org.gradle.api.Project;
 import org.gradle.api.Task;
 import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.component.ModuleComponentIdentifier;
 import org.gradle.api.tasks.TaskProvider;
 
 import java.io.File;
 import java.nio.file.Path;
 
+import static org.elasticsearch.gradle.internal.util.DependenciesUtils.createFileCollectionFromNonTransitiveArtifactsView;
+
 public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin {
 
     public static final String JDK_JAR_HELL_CONFIG_NAME = "jdkJarHell";
@@ -54,12 +57,14 @@ public TaskProvider<? extends Task> createTask(Project project) {
             Configuration compileOnly = project.getConfigurations()
                 .getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
             t.setClasspath(runtimeConfiguration.plus(compileOnly));
-            t.getJarsToScan().from(runtimeConfiguration.fileCollection(dep -> {
-                // These are SelfResolvingDependency, and some of them backed by file collections, like the Gradle API files,
-                // or dependencies added as `files(...)`, we can't be sure if those are third party or not.
-                // err on the side of scanning these to make sure we don't miss anything
-                return dep.getGroup() != null && dep.getGroup().startsWith("org.elasticsearch") == false;
-            }));
+            t.getJarsToScan()
+                .from(
+                    createFileCollectionFromNonTransitiveArtifactsView(
+                        runtimeConfiguration,
+                        identifier -> identifier instanceof ModuleComponentIdentifier
+                            && ((ModuleComponentIdentifier) identifier).getGroup().startsWith("org.elasticsearch") == false
+                    )
+                );
             t.dependsOn(resourcesTask);
             if (BuildParams.getIsRuntimeJavaHomeSet()) {
                 t.getJavaHome().set(project.provider(BuildParams::getRuntimeJavaHome).map(File::getPath));

diff --git a/...ools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java b/...ools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.gradle.internal.util;
+
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.artifacts.ResolvableDependencies;
+import org.gradle.api.artifacts.component.ComponentIdentifier;
+import org.gradle.api.artifacts.result.ResolvedComponentResult;
+import org.gradle.api.artifacts.result.ResolvedDependencyResult;
+import org.gradle.api.file.FileCollection;
+import org.gradle.api.specs.AndSpec;
+import org.gradle.api.specs.Spec;
+
+import java.util.Set;
+import java.util.stream.Collectors;
+
+public class DependenciesUtils {
+
+    public static FileCollection createFileCollectionFromNonTransitiveArtifactsView(
+        Configuration configuration,
+        Spec<ComponentIdentifier> componentFilter
+    ) {
+        ResolvableDependencies incoming = configuration.getIncoming();
+        return incoming.artifactView(viewConfiguration -> {
+            Set<ComponentIdentifier> firstLevelDependencyComponents = incoming.getResolutionResult()
+                .getRootComponent()
+                .map(
+                    rootComponent -> rootComponent.getDependencies()
+                        .stream()
+                        .filter(dependency -> dependency instanceof ResolvedDependencyResult)
+                        .map(dependency -> (ResolvedDependencyResult) dependency)
+                        .filter(dependency -> dependency.getSelected() instanceof ResolvedComponentResult)
+                        .map(dependency -> dependency.getSelected().getId())
+                        .collect(Collectors.toSet())
+                )
+                .get();
+            viewConfiguration.componentFilter(
+                new AndSpec<>(identifier -> firstLevelDependencyComponents.contains(identifier), componentFilter)
+            );
+        }).getFiles();
+    }
+
+}
diff --git a/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/HdfsUtils.java b/build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/HdfsUtils.java
diff --git a/build-tools-internal/src/main/resources/minimumGradleVersion b/build-tools-internal/src/main/resources/minimumGradleVersion
@@ -1 +1 @@
-8.7
+8.8