diff --git a/.baseline/checkstyle/checkstyle-suppressions.xml b/.baseline/checkstyle/checkstyle-suppressions.xml
index f17b9145ec27c..de762b16b0473 100644
--- a/.baseline/checkstyle/checkstyle-suppressions.xml
+++ b/.baseline/checkstyle/checkstyle-suppressions.xml
@@ -12,6 +12,11 @@
+
+
+
+
+
diff --git a/.baseline/checkstyle/checkstyle.xml b/.baseline/checkstyle/checkstyle.xml
index 23938c9fcfe17..86b2e624468b8 100644
--- a/.baseline/checkstyle/checkstyle.xml
+++ b/.baseline/checkstyle/checkstyle.xml
@@ -124,11 +124,17 @@
-
+
+
+
+
+
+
-
+
+
@@ -148,6 +154,10 @@
+
+
+
+
@@ -171,17 +181,15 @@
-
-
+
+
+
-
-
-
-
-
-
+
+
+
@@ -272,6 +280,11 @@
+
+
+
+
+
@@ -322,6 +335,18 @@
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/.baseline/copyright/001_apache-2.0.txt b/.baseline/copyright/001_apache-2.0.txt
index 51fca54c2a05e..54d4a0312b565 100644
--- a/.baseline/copyright/001_apache-2.0.txt
+++ b/.baseline/copyright/001_apache-2.0.txt
@@ -1,3 +1,5 @@
+(c) Copyright ${today.year} Palantir Technologies Inc. All rights reserved.
+
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
diff --git a/.baseline/idea/intellij-java-palantir-style.xml b/.baseline/idea/intellij-java-palantir-style.xml
index 12be2578feda6..fbfbd5c4963cd 100644
--- a/.baseline/idea/intellij-java-palantir-style.xml
+++ b/.baseline/idea/intellij-java-palantir-style.xml
@@ -50,6 +50,7 @@
+
@@ -85,31 +86,39 @@
+
-
+
+
+
-
+
+
+
+
-
+
+
+
-
+
diff --git a/build.gradle b/build.gradle
index 0450cb1fed3f5..1c8a73794df0b 100644
--- a/build.gradle
+++ b/build.gradle
@@ -23,42 +23,23 @@ buildscript {
}
dependencies {
- classpath 'com.palantir.baseline:gradle-baseline-java:0.29.2'
+ classpath 'com.palantir.baseline:gradle-baseline-java:0.41.0'
+ classpath 'com.palantir.gradle.gitversion:gradle-git-version:0.11.0'
classpath 'com.jfrog.bintray.gradle:gradle-bintray-plugin:1.8.4'
- classpath 'com.netflix.nebula:nebula-dependency-recommender:5.2.0'
- classpath 'com.netflix.nebula:nebula-publishing-plugin:5.1.5'
+ classpath 'com.netflix.nebula:nebula-publishing-plugin:9.2.0'
}
}
-plugins {
- id 'com.palantir.git-version' version '0.9.1'
- id "org.inferred.processors" version "1.2.16"
-}
-
-apply plugin: 'com.palantir.baseline-config' // provides baselineUpdateConfig task
-apply plugin: 'com.palantir.baseline-idea'
apply plugin: 'com.palantir.baseline'
-apply plugin: 'com.palantir.baseline-class-uniqueness'
-
-dependencies {
- baseline "com.palantir.baseline:gradle-baseline-java-config:0.29.2@zip"
-}
+apply plugin: 'com.palantir.git-version'
allprojects {
group 'org.apache.spark'
version System.env.CIRCLE_TAG ?: gitVersion()
buildDir = 'gradle-build'
- apply plugin: 'idea'
- apply plugin: 'nebula.dependency-recommender'
-
repositories {
jcenter()
maven { url "http://palantir.bintray.com/releases" }
}
-
- dependencyRecommendations {
- strategy OverrideTransitives
- propertiesFile file: project.rootProject.file('versions.props')
- }
}
diff --git a/spark-docker-image-generator/build.gradle b/spark-docker-image-generator/build.gradle
index 38df4f4e252b3..770a4652ab667 100644
--- a/spark-docker-image-generator/build.gradle
+++ b/spark-docker-image-generator/build.gradle
@@ -15,15 +15,12 @@
* limitations under the License.
*/
-plugins {
- id 'java-gradle-plugin'
-}
-
-apply plugin: 'java'
-apply plugin: 'idea'
+apply plugin: 'java-gradle-plugin'
apply from: "${rootDir}/gradle/publish.gradle"
-sourceCompatibility = 1.8
+java {
+ sourceCompatibility JavaVersion.VERSION_1_8
+}
dependencies {
compile gradleApi()
@@ -36,11 +33,6 @@ dependencies {
testCompile gradleTestKit()
}
-test {
- minHeapSize = "512m"
- maxHeapSize = "512m"
-}
-
task prepareDockerBundleDir(type: Zip) {
from("${rootDir}/resource-managers/kubernetes/docker/src/main/dockerfiles/spark/Dockerfile") {
into 'kubernetes/dockerfiles/spark'
@@ -83,6 +75,8 @@ publishing {
test {
systemProperty 'docker-resources-zip-path', "${buildDir}/docker-resources/docker-resources.zip"
+ minHeapSize = "512m"
+ maxHeapSize = "512m"
}
tasks.test.dependsOn tasks.publishDockerBundlePublicationToMavenTestRepository
diff --git a/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/DockerBuildTask.java b/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/DockerBuildTask.java
index 9ed755695ffce..0984a2e117c09 100644
--- a/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/DockerBuildTask.java
+++ b/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/DockerBuildTask.java
@@ -17,7 +17,6 @@
package org.apache.spark.deploy.kubernetes.docker.gradle;
import java.io.File;
-
import org.gradle.api.DefaultTask;
import org.gradle.api.provider.Property;
import org.gradle.api.tasks.Input;
@@ -25,49 +24,49 @@
import org.gradle.api.tasks.InputFile;
import org.gradle.api.tasks.TaskAction;
-public class DockerBuildTask extends DefaultTask {
+public class DockerBuildTask extends DefaultTask {
- private File dockerFile;
- private File dockerBuildDirectory;
- private Property imageName;
+ private File dockerFile;
+ private File dockerBuildDirectory;
+ private Property imageName;
- @InputFile
- public final File getDockerFile() {
- return dockerFile;
- }
+ @InputFile
+ public final File getDockerFile() {
+ return dockerFile;
+ }
- @InputDirectory
- public final File getDockerBuildDirectory() {
- return dockerBuildDirectory;
- }
+ @InputDirectory
+ public final File getDockerBuildDirectory() {
+ return dockerBuildDirectory;
+ }
- @Input
- public final Property getImageName() {
- return imageName;
- }
+ @Input
+ public final Property getImageName() {
+ return imageName;
+ }
- public final void setDockerFile(File dockerFile) {
- this.dockerFile = dockerFile;
- }
+ public final void setDockerFile(File dockerFile) {
+ this.dockerFile = dockerFile;
+ }
- public final void setDockerBuildDirectory(File dockerBuildDirectory) {
- this.dockerBuildDirectory = dockerBuildDirectory;
- }
+ public final void setDockerBuildDirectory(File dockerBuildDirectory) {
+ this.dockerBuildDirectory = dockerBuildDirectory;
+ }
- public final void setImageName(Property imageName) {
- this.imageName = imageName;
- }
+ public final void setImageName(Property imageName) {
+ this.imageName = imageName;
+ }
- @TaskAction
- public final void exec() {
- getProject().exec(execSpec -> execSpec.commandLine(
- "docker",
- "build",
- "-f",
- dockerFile.getAbsolutePath(),
- "-t",
- imageName.get(),
- dockerBuildDirectory.getAbsolutePath()));
- }
+ @TaskAction
+ public final void exec() {
+ getProject().exec(execSpec -> execSpec.commandLine(
+ "docker",
+ "build",
+ "-f",
+ dockerFile.getAbsolutePath(),
+ "-t",
+ imageName.get(),
+ dockerBuildDirectory.getAbsolutePath()));
+ }
}
diff --git a/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/GenerateDockerFileTask.java b/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/GenerateDockerFileTask.java
index ee7ab4bfab5e9..63d9160ee23a4 100644
--- a/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/GenerateDockerFileTask.java
+++ b/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/GenerateDockerFileTask.java
@@ -24,7 +24,6 @@
import java.util.List;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-
import org.gradle.api.DefaultTask;
import org.gradle.api.provider.Property;
import org.gradle.api.tasks.Input;
@@ -34,58 +33,57 @@
public class GenerateDockerFileTask extends DefaultTask {
- private File srcDockerFile;
- private File destDockerFile;
- private Property baseImage;
+ private File srcDockerFile;
+ private File destDockerFile;
+ private Property baseImage;
- public final void setSrcDockerFile(File srcDockerFile) {
- this.srcDockerFile = srcDockerFile;
- }
+ public final void setSrcDockerFile(File srcDockerFile) {
+ this.srcDockerFile = srcDockerFile;
+ }
- public final void setDestDockerFile(File destDockerFile) {
- this.destDockerFile = destDockerFile;
- }
+ public final void setDestDockerFile(File destDockerFile) {
+ this.destDockerFile = destDockerFile;
+ }
- public final void setBaseImage(Property baseImage) {
- this.baseImage = baseImage;
- }
+ public final void setBaseImage(Property baseImage) {
+ this.baseImage = baseImage;
+ }
- @InputFile
- public final File getSrcDockerFile() {
- return srcDockerFile;
- }
+ @InputFile
+ public final File getSrcDockerFile() {
+ return srcDockerFile;
+ }
- @OutputFile
- public final File getDestDockerFile() {
- return destDockerFile;
- }
+ @OutputFile
+ public final File getDestDockerFile() {
+ return destDockerFile;
+ }
- @Input
- public final Property getBaseImage() {
- return baseImage;
- }
+ @Input
+ public final Property getBaseImage() {
+ return baseImage;
+ }
- @TaskAction
- public final void generateDockerFile() throws IOException {
- if (!baseImage.isPresent()) {
- Files.copy(srcDockerFile.toPath(), destDockerFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
- } else {
- File currentSrcDockerFile = getSrcDockerFile();
- File currentDestDockerFile = getDestDockerFile();
- List fileLines;
- try (Stream rawLines = Files.lines(currentSrcDockerFile.toPath(), StandardCharsets.UTF_8)) {
- fileLines = rawLines
- .map(line -> {
- if (line.equals("FROM openjdk:8-alpine")) {
- return String.format("FROM %s", baseImage.get());
- } else {
- return line;
- }
+ @TaskAction
+ public final void generateDockerFile() throws IOException {
+ if (!baseImage.isPresent()) {
+ Files.copy(srcDockerFile.toPath(), destDockerFile.toPath(), StandardCopyOption.REPLACE_EXISTING);
+ } else {
+ File currentSrcDockerFile = getSrcDockerFile();
+ File currentDestDockerFile = getDestDockerFile();
+ List fileLines;
+ try (Stream rawLines = Files.lines(currentSrcDockerFile.toPath(), StandardCharsets.UTF_8)) {
+ fileLines = rawLines.map(line -> {
+ if (line.equals("FROM openjdk:8-alpine")) {
+ return String.format("FROM %s", baseImage.get());
+ } else {
+ return line;
+ }
}).collect(Collectors.toList());
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- Files.write(currentDestDockerFile.toPath(), fileLines, StandardCharsets.UTF_8);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ Files.write(currentDestDockerFile.toPath(), fileLines, StandardCharsets.UTF_8);
+ }
}
- }
}
diff --git a/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/LazyExecTask.java b/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/LazyExecTask.java
index bc2f485f1cee2..6bd8fc73bf1f3 100644
--- a/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/LazyExecTask.java
+++ b/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/LazyExecTask.java
@@ -18,7 +18,6 @@
import java.util.List;
import java.util.stream.Collectors;
-
import org.gradle.api.DefaultTask;
import org.gradle.api.provider.Property;
import org.gradle.api.tasks.Input;
@@ -26,22 +25,22 @@
public class LazyExecTask extends DefaultTask {
- private List> commandLine;
+ private List> commandLine;
- @Input
- public final List> getCommandLine() {
- return commandLine;
- }
+ @Input
+ public final List> getCommandLine() {
+ return commandLine;
+ }
- public final void setCommandLine(List> commandLine) {
- this.commandLine = commandLine;
- }
+ public final void setCommandLine(List> commandLine) {
+ this.commandLine = commandLine;
+ }
- @TaskAction
- public final void runCommand() {
- getProject().exec(exec ->
- exec.commandLine(commandLine.stream()
- .map(Property::get)
- .collect(Collectors.toList())));
- }
+ @TaskAction
+ public final void runCommand() {
+ getProject().exec(exec ->
+ exec.commandLine(commandLine.stream()
+ .map(Property::get)
+ .collect(Collectors.toList())));
+ }
}
diff --git a/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/SparkDockerExtension.java b/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/SparkDockerExtension.java
index aedf48e54abea..8b3dbf6ad8b52 100644
--- a/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/SparkDockerExtension.java
+++ b/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/SparkDockerExtension.java
@@ -19,52 +19,51 @@
import java.util.Collection;
import java.util.stream.Collectors;
import java.util.stream.Stream;
-
import org.gradle.api.Project;
import org.gradle.api.provider.Property;
import org.gradle.api.provider.SetProperty;
public class SparkDockerExtension {
- private final Property baseImage;
- private final Property imageName;
- private final SetProperty tags;
+ private final Property baseImage;
+ private final Property imageName;
+ private final SetProperty tags;
- public SparkDockerExtension(Project project) {
- this.baseImage = project.getObjects().property(String.class);
- this.imageName = project.getObjects().property(String.class);
- this.tags = project.getObjects().setProperty(String.class);
- }
+ public SparkDockerExtension(Project project) {
+ this.baseImage = project.getObjects().property(String.class);
+ this.imageName = project.getObjects().property(String.class);
+ this.tags = project.getObjects().setProperty(String.class);
+ }
- public final Property getBaseImage() {
- return baseImage;
- }
+ public final Property getBaseImage() {
+ return baseImage;
+ }
- public final Property getImageName() {
- return imageName;
- }
+ public final Property getImageName() {
+ return imageName;
+ }
- public final SetProperty getTags() {
- return tags;
- }
+ public final SetProperty getTags() {
+ return tags;
+ }
- @SuppressWarnings("HiddenField")
- public final void baseImage(String baseImage) {
- this.baseImage.set(baseImage);
- }
+ @SuppressWarnings("HiddenField")
+ public final void baseImage(String baseImage) {
+ this.baseImage.set(baseImage);
+ }
- @SuppressWarnings("HiddenField")
- public final void imageName(String imageName) {
- this.imageName.set(imageName);
- }
+ @SuppressWarnings("HiddenField")
+ public final void imageName(String imageName) {
+ this.imageName.set(imageName);
+ }
- @SuppressWarnings("HiddenField")
- public final void tags(Collection tags) {
- this.tags.set(tags);
- }
+ @SuppressWarnings("HiddenField")
+ public final void tags(Collection tags) {
+ this.tags.set(tags);
+ }
- @SuppressWarnings("HiddenField")
- public final void tags(String... tags) {
- this.tags.set(Stream.of(tags).collect(Collectors.toSet()));
- }
+ @SuppressWarnings("HiddenField")
+ public final void tags(String... tags) {
+ this.tags.set(Stream.of(tags).collect(Collectors.toSet()));
+ }
}
diff --git a/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/SparkDockerPlugin.java b/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/SparkDockerPlugin.java
index b1ade9cb509bf..915d94065438b 100644
--- a/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/SparkDockerPlugin.java
+++ b/spark-docker-image-generator/src/main/java/org/apache/spark/deploy/kubernetes/docker/gradle/SparkDockerPlugin.java
@@ -24,7 +24,6 @@
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
-
import org.gradle.api.Plugin;
import org.gradle.api.Project;
import org.gradle.api.Task;
@@ -38,144 +37,145 @@
import org.slf4j.LoggerFactory;
public final class SparkDockerPlugin implements Plugin {
- private static final Logger log = LoggerFactory.getLogger(SparkDockerPlugin.class);
+ private static final Logger log = LoggerFactory.getLogger(SparkDockerPlugin.class);
- public static final String DOCKER_IMAGE_EXTENSION = "sparkDocker";
- public static final String SPARK_JARS_DIR = "jars";
- public static final String SPARK_DOCKER_RUNTIME_CONFIGURATION_NAME = "sparkDockerRuntime";
+ public static final String DOCKER_IMAGE_EXTENSION = "sparkDocker";
+ public static final String SPARK_JARS_DIR = "jars";
+ public static final String SPARK_DOCKER_RUNTIME_CONFIGURATION_NAME = "sparkDockerRuntime";
- @Override
- public void apply(Project project) {
- File dockerBuildDirectory = new File(project.getBuildDir(), "spark-docker-build");
- SparkDockerExtension extension = project.getExtensions().create(
- DOCKER_IMAGE_EXTENSION, SparkDockerExtension.class, project);
- if (!dockerBuildDirectory.isDirectory() && !dockerBuildDirectory.mkdirs()) {
- throw new RuntimeException("Failed to create Docker build directory at "
- + dockerBuildDirectory.getAbsolutePath());
+ @Override
+ public void apply(Project project) {
+ File dockerBuildDirectory = new File(project.getBuildDir(), "spark-docker-build");
+ SparkDockerExtension extension = project.getExtensions().create(
+ DOCKER_IMAGE_EXTENSION, SparkDockerExtension.class, project);
+ if (!dockerBuildDirectory.isDirectory() && !dockerBuildDirectory.mkdirs()) {
+ throw new RuntimeException("Failed to create Docker build directory at "
+ + dockerBuildDirectory.getAbsolutePath());
+ }
+ Configuration sparkDockerRuntimeConfiguration =
+ project.getConfigurations().maybeCreate(SPARK_DOCKER_RUNTIME_CONFIGURATION_NAME);
+ File dockerFile = new File(dockerBuildDirectory, "Dockerfile");
+ project.getPluginManager().withPlugin("java", plugin -> {
+ Configuration runtimeConfiguration = project.getConfigurations().findByName("runtime");
+ if (runtimeConfiguration == null) {
+ log.warn("No runtime configuration was found for a reference configuration for building"
+ + " your Spark application's docker images.");
+ } else {
+ sparkDockerRuntimeConfiguration.extendsFrom(runtimeConfiguration);
+ }
+ Task jarTask = project.getTasks().named("jar").get();
+ Provider sparkAppJar = project.getProviders().provider(() ->
+ jarTask.getOutputs().getFiles().getSingleFile());
+ Provider jarsDirProvider = project.getProviders().provider(() ->
+ new File(dockerBuildDirectory, SPARK_JARS_DIR));
+ Task copySparkAppLibTask = project.getTasks().create(
+ "copySparkAppLibIntoDocker",
+ Copy.class,
+ copyTask -> copyTask.from(
+ project.files(project.getConfigurations()
+ .getByName(SPARK_DOCKER_RUNTIME_CONFIGURATION_NAME)),
+ sparkAppJar)
+ .into(jarsDirProvider));
+ copySparkAppLibTask.dependsOn(jarTask);
+ String version = Optional.ofNullable(getClass().getPackage().getImplementationVersion())
+ .orElse("latest.release");
+ Configuration dockerResourcesConf = project.getConfigurations().detachedConfiguration(
+ project.getDependencies().create("org.apache.spark:spark-docker-resources:" + version));
+ Sync deployScriptsTask = project.getTasks().create(
+ "sparkDockerDeployScripts", Sync.class, task -> {
+ task.from(project.zipTree(dockerResourcesConf.getSingleFile()));
+ task.setIncludeEmptyDirs(false);
+ task.into(dockerBuildDirectory);
+ });
+ copySparkAppLibTask.dependsOn(deployScriptsTask);
+ GenerateDockerFileTask generateDockerFileTask = project.getTasks().create(
+ "sparkDockerGenerateDockerFile", GenerateDockerFileTask.class);
+ generateDockerFileTask.setSrcDockerFile(
+ Paths.get(
+ dockerBuildDirectory.getAbsolutePath(),
+ "kubernetes",
+ "dockerfiles",
+ "spark",
+ "Dockerfile.original").toFile());
+ generateDockerFileTask.setDestDockerFile(dockerFile);
+ generateDockerFileTask.dependsOn(deployScriptsTask);
+ generateDockerFileTask.setBaseImage(extension.getBaseImage());
+ Task prepareTask = project.getTasks().create("sparkDockerPrepare");
+ prepareTask.dependsOn(
+ deployScriptsTask, generateDockerFileTask, "copySparkAppLibIntoDocker");
+ });
+ setupDockerTasks(dockerBuildDirectory, dockerFile, project, extension);
}
- Configuration sparkDockerRuntimeConfiguration =
- project.getConfigurations().maybeCreate(SPARK_DOCKER_RUNTIME_CONFIGURATION_NAME);
- File dockerFile = new File(dockerBuildDirectory, "Dockerfile");
- project.getPluginManager().withPlugin("java", plugin -> {
- Configuration runtimeConfiguration = project.getConfigurations().findByName("runtime");
- if (runtimeConfiguration == null) {
- log.warn("No runtime configuration was found for a reference configuration for building"
- + " your Spark application's docker images.");
- } else {
- sparkDockerRuntimeConfiguration.extendsFrom(runtimeConfiguration);
- }
- Task jarTask = project.getTasks().named("jar").get();
- Provider sparkAppJar = project.getProviders().provider(() ->
- jarTask.getOutputs().getFiles().getSingleFile());
- Provider jarsDirProvider = project.getProviders().provider(() ->
- new File(dockerBuildDirectory, SPARK_JARS_DIR));
- Task copySparkAppLibTask = project.getTasks().create(
- "copySparkAppLibIntoDocker",
- Copy.class,
- copyTask -> copyTask.from(
- project.files(project.getConfigurations().getByName(SPARK_DOCKER_RUNTIME_CONFIGURATION_NAME)),
- sparkAppJar)
- .into(jarsDirProvider));
- copySparkAppLibTask.dependsOn(jarTask);
- String version = Optional.ofNullable(getClass().getPackage().getImplementationVersion())
- .orElse("latest.release");
- Configuration dockerResourcesConf = project.getConfigurations().detachedConfiguration(
- project.getDependencies().create("org.apache.spark:spark-docker-resources:" + version));
- Sync deployScriptsTask = project.getTasks().create(
- "sparkDockerDeployScripts", Sync.class, task -> {
- task.from(project.zipTree(dockerResourcesConf.getSingleFile()));
- task.setIncludeEmptyDirs(false);
- task.into(dockerBuildDirectory);
- });
- copySparkAppLibTask.dependsOn(deployScriptsTask);
- GenerateDockerFileTask generateDockerFileTask = project.getTasks().create(
- "sparkDockerGenerateDockerFile", GenerateDockerFileTask.class);
- generateDockerFileTask.setSrcDockerFile(
- Paths.get(
- dockerBuildDirectory.getAbsolutePath(),
- "kubernetes",
- "dockerfiles",
- "spark",
- "Dockerfile.original").toFile());
- generateDockerFileTask.setDestDockerFile(dockerFile);
- generateDockerFileTask.dependsOn(deployScriptsTask);
- generateDockerFileTask.setBaseImage(extension.getBaseImage());
- Task prepareTask = project.getTasks().create("sparkDockerPrepare");
- prepareTask.dependsOn(
- deployScriptsTask, generateDockerFileTask, "copySparkAppLibIntoDocker");
- });
- setupDockerTasks(dockerBuildDirectory, dockerFile, project, extension);
- }
- private void setupDockerTasks(
- File buildDirectory,
- File dockerFile,
- Project project,
- SparkDockerExtension extension) {
- DockerBuildTask dockerBuild = project.getTasks().create(
- "sparkDockerBuild",
- DockerBuildTask.class,
- dockerBuildTask -> {
- dockerBuildTask.setDockerBuildDirectory(buildDirectory);
- dockerBuildTask.setDockerFile(dockerFile);
- dockerBuildTask.setImageName(extension.getImageName());
- dockerBuildTask.dependsOn("sparkDockerPrepare");
- });
- Task tagAllTask = project.getTasks().create("sparkDockerTag");
- LazyExecTask pushAllTask = project.getTasks().create(
- "sparkDockerPush",
- LazyExecTask.class,
- task -> {
- List> commandLine = new ArrayList<>();
- commandLine.add(constProperty(project, "docker"));
- commandLine.add(constProperty(project, "push"));
- commandLine.add(extension.getImageName());
- task.setCommandLine(commandLine);
+ private void setupDockerTasks(
+ File buildDirectory,
+ File dockerFile,
+ Project project,
+ SparkDockerExtension extension) {
+ DockerBuildTask dockerBuild = project.getTasks().create(
+ "sparkDockerBuild",
+ DockerBuildTask.class,
+ dockerBuildTask -> {
+ dockerBuildTask.setDockerBuildDirectory(buildDirectory);
+ dockerBuildTask.setDockerFile(dockerFile);
+ dockerBuildTask.setImageName(extension.getImageName());
+ dockerBuildTask.dependsOn("sparkDockerPrepare");
+ });
+ Task tagAllTask = project.getTasks().create("sparkDockerTag");
+ LazyExecTask pushAllTask = project.getTasks().create(
+ "sparkDockerPush",
+ LazyExecTask.class,
+ task -> {
+ List> commandLine = new ArrayList<>();
+ commandLine.add(constProperty(project, "docker"));
+ commandLine.add(constProperty(project, "push"));
+ commandLine.add(extension.getImageName());
+ task.setCommandLine(commandLine);
+ });
+ project.afterEvaluate(evaluatedProject -> {
+ Set tags = extension.getTags().getOrElse(Collections.emptySet());
+ String resolvedImageName = extension.getImageName().get();
+ List tagTasks = tags.stream()
+ .map(tag ->
+ evaluatedProject
+ .getTasks()
+ .create(
+ String.format("sparkDockerTag%s", tag),
+ Exec.class,
+ task ->
+ task.commandLine(
+ "docker",
+ "tag",
+ resolvedImageName,
+ String.format("%s:%s", resolvedImageName, tag))
+ .dependsOn(dockerBuild)))
+ .collect(Collectors.toList());
+ if (!tagTasks.isEmpty()) {
+ tagAllTask.dependsOn(tagTasks);
+ } else {
+ tagAllTask.dependsOn(dockerBuild);
+ }
+ List pushTasks = tags.stream()
+ .map(tag ->
+ evaluatedProject.getTasks().create(
+ String.format("sparkDockerPush%s", tag),
+ Exec.class,
+ task ->
+ task.commandLine(
+ "docker", "push", String.format("%s:%s", resolvedImageName, tag))
+ .dependsOn(String.format("sparkDockerTag%s", tag))))
+ .collect(Collectors.toList());
+ if (!pushTasks.isEmpty()) {
+ pushAllTask.dependsOn(pushTasks);
+ } else {
+ pushAllTask.dependsOn(tagAllTask);
+ }
});
- project.afterEvaluate(evaluatedProject -> {
- Set tags = extension.getTags().getOrElse(Collections.emptySet());
- String resolvedImageName = extension.getImageName().get();
- List tagTasks = tags.stream()
- .map(tag ->
- evaluatedProject
- .getTasks()
- .create(
- String.format("sparkDockerTag%s", tag),
- Exec.class,
- task ->
- task.commandLine(
- "docker",
- "tag",
- resolvedImageName,
- String.format("%s:%s", resolvedImageName, tag))
- .dependsOn(dockerBuild)))
- .collect(Collectors.toList());
- if (!tagTasks.isEmpty()) {
- tagAllTask.dependsOn(tagTasks);
- } else {
- tagAllTask.dependsOn(dockerBuild);
- }
- List pushTasks = tags.stream()
- .map(tag ->
- evaluatedProject.getTasks().create(
- String.format("sparkDockerPush%s", tag),
- Exec.class,
- task ->
- task.commandLine(
- "docker", "push", String.format("%s:%s", resolvedImageName, tag))
- .dependsOn(String.format("sparkDockerTag%s", tag))))
- .collect(Collectors.toList());
- if (!pushTasks.isEmpty()) {
- pushAllTask.dependsOn(pushTasks);
- } else {
- pushAllTask.dependsOn(tagAllTask);
- }
- });
- }
+ }
- private Property constProperty(Project project, String value) {
- Property prop = project.getObjects().property(String.class);
- prop.set(value);
- return prop;
- }
+ private Property constProperty(Project project, String value) {
+ Property prop = project.getObjects().property(String.class);
+ prop.set(value);
+ return prop;
+ }
}
diff --git a/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/DockerBuildTaskSuite.java b/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/DockerBuildTaskSuite.java
index 570515c6a989c..c03b31f488157 100644
--- a/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/DockerBuildTaskSuite.java
+++ b/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/DockerBuildTaskSuite.java
@@ -18,7 +18,6 @@
import java.io.File;
import java.io.IOException;
-
import org.gradle.api.Project;
import org.gradle.api.provider.Property;
import org.gradle.process.ExecSpec;
@@ -34,50 +33,50 @@
@RunWith(MockitoJUnitRunner.class)
public final class DockerBuildTaskSuite {
- private static final String IMAGE_NAME = "spark-test";
+ private static final String IMAGE_NAME = "spark-test";
- @Rule
- public TemporaryFolder tempFolder = new TemporaryFolder();
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
- private File dockerFile;
+ private File dockerFile;
- private File dockerBuildDirectory;
+ private File dockerBuildDirectory;
- @Mock
- private Project project;
+ @Mock
+ private Project project;
- @Mock
- private ExecSpec execSpec;
+ @Mock
+ private ExecSpec execSpec;
- @Mock
- private DockerBuildTask taskUnderTest;
+ @Mock
+ private DockerBuildTask taskUnderTest;
- @Mock
- private Property imageName;
+ @Mock
+ private Property imageName;
- @Before
- public void before() throws IOException {
- ProjectExecUtils.invokeExecSpecAction(project, execSpec);
- dockerFile = tempFolder.newFile("Dockerfile");
- dockerBuildDirectory = tempFolder.newFolder("docker-build-dir");
- taskUnderTest.setDockerFile(dockerFile);
- taskUnderTest.setDockerBuildDirectory(dockerBuildDirectory);
- taskUnderTest.setImageName(imageName);
- Mockito.when(imageName.get()).thenReturn(IMAGE_NAME);
- Mockito.when(taskUnderTest.getProject()).thenReturn(project);
- }
+ @Before
+ public void before() throws IOException {
+ ProjectExecUtils.invokeExecSpecAction(project, execSpec);
+ dockerFile = tempFolder.newFile("Dockerfile");
+ dockerBuildDirectory = tempFolder.newFolder("docker-build-dir");
+ taskUnderTest.setDockerFile(dockerFile);
+ taskUnderTest.setDockerBuildDirectory(dockerBuildDirectory);
+ taskUnderTest.setImageName(imageName);
+ Mockito.when(imageName.get()).thenReturn(IMAGE_NAME);
+ Mockito.when(taskUnderTest.getProject()).thenReturn(project);
+ }
- @Test
- public void testExec_setsCommandLineToBuildImage() {
- taskUnderTest.exec();
- Mockito.verify(execSpec).commandLine(
- "docker",
- "build",
- "-f",
- dockerFile.getAbsolutePath(),
- "-t",
- IMAGE_NAME,
- dockerBuildDirectory.getAbsolutePath());
- }
+ @Test
+ public void testExec_setsCommandLineToBuildImage() {
+ taskUnderTest.exec();
+ Mockito.verify(execSpec).commandLine(
+ "docker",
+ "build",
+ "-f",
+ dockerFile.getAbsolutePath(),
+ "-t",
+ IMAGE_NAME,
+ dockerBuildDirectory.getAbsolutePath());
+ }
}
diff --git a/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/GenerateDockerFileTaskSuite.java b/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/GenerateDockerFileTaskSuite.java
index 6b94fcefdafba..7c0ea38953a2e 100644
--- a/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/GenerateDockerFileTaskSuite.java
+++ b/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/GenerateDockerFileTaskSuite.java
@@ -29,7 +29,6 @@
import java.util.stream.Collectors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;
-
import org.apache.commons.io.IOUtils;
import org.assertj.core.api.Assertions;
import org.gradle.api.provider.Property;
@@ -40,63 +39,61 @@
import org.mockito.Mockito;
public final class GenerateDockerFileTaskSuite {
- @Rule
- public TemporaryFolder tempFolder = new TemporaryFolder();
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
- private File srcDockerFile;
- private File destDockerFile;
+ private File srcDockerFile;
+ private File destDockerFile;
- @Before
- public void before() throws IOException {
- File dockerFileDir = tempFolder.newFolder("docker");
- destDockerFile = new File(dockerFileDir, "Dockerfile");
- srcDockerFile = tempFolder.newFile("Dockerfile.original");
+ @Before
+ public void before() throws IOException {
+ File dockerFileDir = tempFolder.newFolder("docker");
+ destDockerFile = new File(dockerFileDir, "Dockerfile");
+ srcDockerFile = tempFolder.newFile("Dockerfile.original");
+ File dockerResourcesFile = new File(System.getProperty("docker-resources-zip-path"));
- try (InputStream originalDockerBundleZipped = new FileInputStream(
- new File(System.getProperty("docker-resources-zip-path")));
- ZipInputStream unzipped = new ZipInputStream(originalDockerBundleZipped);
- FileOutputStream srcDockerFileStream = new FileOutputStream(srcDockerFile)) {
- ZipEntry currentEntry = unzipped.getNextEntry();
- boolean foundDockerFile = false;
- while (currentEntry != null && !foundDockerFile) {
- if (currentEntry.getName().equals("kubernetes/dockerfiles/spark/Dockerfile.original")) {
- IOUtils.copy(unzipped, srcDockerFileStream);
- foundDockerFile = true;
- } else {
- currentEntry = unzipped.getNextEntry();
+ try (InputStream originalDockerBundleZipped = new FileInputStream(dockerResourcesFile);
+ ZipInputStream unzipped = new ZipInputStream(originalDockerBundleZipped);
+ FileOutputStream srcDockerFileStream = new FileOutputStream(srcDockerFile)) {
+ ZipEntry currentEntry = unzipped.getNextEntry();
+ boolean foundDockerFile = false;
+ while (currentEntry != null && !foundDockerFile) {
+ if (currentEntry.getName().equals("kubernetes/dockerfiles/spark/Dockerfile.original")) {
+ IOUtils.copy(unzipped, srcDockerFileStream);
+ foundDockerFile = true;
+ } else {
+ currentEntry = unzipped.getNextEntry();
+ }
+ }
+ if (!foundDockerFile) {
+ throw new IllegalStateException("Dockerfile not found.");
+ }
}
- }
- if (!foundDockerFile) {
- throw new IllegalStateException("Dockerfile not found.");
- }
}
- }
- @Test
- public void testGenerateDockerFile() throws IOException {
- GenerateDockerFileTask task = Mockito.mock(GenerateDockerFileTask.class);
- task.setDestDockerFile(destDockerFile);
- task.setSrcDockerFile(srcDockerFile);
- Property baseImageProperty = Mockito.mock(Property.class);
- Mockito.when(baseImageProperty.isPresent()).thenReturn(true);
- Mockito.when(baseImageProperty.get()).thenReturn("fabric8/java-centos-openjdk8-jdk:latest");
- task.setBaseImage(baseImageProperty);
- task.generateDockerFile();
- Assertions.assertThat(destDockerFile).isFile();
- List writtenLines = Files.readAllLines(
- destDockerFile.toPath(), StandardCharsets.UTF_8);
- try (InputStream expectedDockerFileInput =
- getClass().getResourceAsStream("/ExpectedDockerfile");
- InputStreamReader expectedDockerFileReader =
- new InputStreamReader(expectedDockerFileInput, StandardCharsets.UTF_8);
- BufferedReader expectedDockerFileBuffered =
- new BufferedReader(expectedDockerFileReader)) {
- List expectedFileLines = expectedDockerFileBuffered
- .lines()
- .collect(Collectors.toList());
- Assertions.assertThat(writtenLines).isEqualTo(expectedFileLines);
+ @Test
+ public void testGenerateDockerFile() throws IOException {
+ GenerateDockerFileTask task = Mockito.mock(GenerateDockerFileTask.class);
+ task.setDestDockerFile(destDockerFile);
+ task.setSrcDockerFile(srcDockerFile);
+ Property baseImageProperty = Mockito.mock(Property.class);
+ Mockito.when(baseImageProperty.isPresent()).thenReturn(true);
+ Mockito.when(baseImageProperty.get()).thenReturn("fabric8/java-centos-openjdk8-jdk:latest");
+ task.setBaseImage(baseImageProperty);
+ task.generateDockerFile();
+ Assertions.assertThat(destDockerFile).isFile();
+ List writtenLines = Files.readAllLines(
+ destDockerFile.toPath(), StandardCharsets.UTF_8);
+ try (InputStream expectedDockerFileInput = getClass().getResourceAsStream("/ExpectedDockerfile");
+ InputStreamReader expectedDockerFileReader =
+ new InputStreamReader(expectedDockerFileInput, StandardCharsets.UTF_8);
+ BufferedReader expectedDockerFileBuffered =
+ new BufferedReader(expectedDockerFileReader)) {
+ List expectedFileLines = expectedDockerFileBuffered
+ .lines()
+ .collect(Collectors.toList());
+ Assertions.assertThat(writtenLines).isEqualTo(expectedFileLines);
+ }
}
- }
}
-
diff --git a/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/LazyExecTaskSuite.java b/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/LazyExecTaskSuite.java
index 1fffbf33abcf2..4f0240244fe57 100644
--- a/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/LazyExecTaskSuite.java
+++ b/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/LazyExecTaskSuite.java
@@ -18,7 +18,6 @@
import java.util.ArrayList;
import java.util.List;
-
import org.gradle.api.Project;
import org.gradle.api.provider.Property;
import org.gradle.process.ExecSpec;
@@ -32,39 +31,39 @@
@RunWith(MockitoJUnitRunner.class)
public final class LazyExecTaskSuite {
- @Mock
- private LazyExecTask taskUnderTest;
+ @Mock
+ private org.apache.spark.deploy.kubernetes.docker.gradle.LazyExecTask taskUnderTest;
- @Mock
- private Project project;
+ @Mock
+ private Project project;
- @Mock
- private ExecSpec execSpec;
+ @Mock
+ private ExecSpec execSpec;
- @Before
- public void before() {
- ProjectExecUtils.invokeExecSpecAction(project, execSpec);
- Mockito.when(taskUnderTest.getProject()).thenReturn(project);
- }
+ @Before
+ public void before() {
+ org.apache.spark.deploy.kubernetes.docker.gradle.ProjectExecUtils.invokeExecSpecAction(project, execSpec);
+ Mockito.when(taskUnderTest.getProject()).thenReturn(project);
+ }
- @Test
- public void testRunCommand_setsCommandLineOnExecSpec() {
- List> command = new ArrayList<>();
- command.add(constProperty("ls"));
- command.add(constProperty("-lahrt"));
- command.add(constProperty("git"));
- List expectedCommand = new ArrayList<>();
- expectedCommand.add("ls");
- expectedCommand.add("-lahrt");
- expectedCommand.add("git");
- taskUnderTest.setCommandLine(command);
- taskUnderTest.runCommand();
- Mockito.verify(execSpec).commandLine(expectedCommand);
- }
+ @Test
+ public void testRunCommand_setsCommandLineOnExecSpec() {
+ List> command = new ArrayList<>();
+ command.add(constProperty("ls"));
+ command.add(constProperty("-lahrt"));
+ command.add(constProperty("git"));
+ List expectedCommand = new ArrayList<>();
+ expectedCommand.add("ls");
+ expectedCommand.add("-lahrt");
+ expectedCommand.add("git");
+ taskUnderTest.setCommandLine(command);
+ taskUnderTest.runCommand();
+ Mockito.verify(execSpec).commandLine(expectedCommand);
+ }
- private Property constProperty(T value) {
- Property property = (Property) Mockito.mock(Property.class);
- Mockito.when(property.get()).thenReturn(value);
- return property;
- }
+ private Property constProperty(T value) {
+ Property property = (Property) Mockito.mock(Property.class);
+ Mockito.when(property.get()).thenReturn(value);
+ return property;
+ }
}
diff --git a/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/ProjectExecUtils.java b/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/ProjectExecUtils.java
index 45c639363f8da..ec8a68098855b 100644
--- a/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/ProjectExecUtils.java
+++ b/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/ProjectExecUtils.java
@@ -23,12 +23,13 @@
public final class ProjectExecUtils {
- public static void invokeExecSpecAction(Project project, ExecSpec execSpec) {
- Mockito.when(project.exec(Mockito.any(Action.class))).thenAnswer(invocation -> {
- ((Action) invocation.getArgument(0)).execute(execSpec);
- return null;
- });
- }
+ public static void invokeExecSpecAction(Project project, ExecSpec execSpec) {
+ Mockito.when(project.exec(Mockito.any(Action.class))).thenAnswer(invocation -> {
+ ((Action) invocation.getArgument(0)).execute(execSpec);
+ return null;
+ });
+ }
- private ProjectExecUtils() {}
+ private ProjectExecUtils() {
+ }
}
diff --git a/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/SparkDockerPluginSuite.java b/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/SparkDockerPluginSuite.java
index 6045e8dc095b8..39f85184d6db8 100644
--- a/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/SparkDockerPluginSuite.java
+++ b/spark-docker-image-generator/src/test/java/org/apache/spark/deploy/kubernetes/docker/gradle/SparkDockerPluginSuite.java
@@ -16,10 +16,6 @@
*/
package org.apache.spark.deploy.kubernetes.docker.gradle;
-import java.io.File;
-import java.nio.file.Paths;
-import java.util.UUID;
-
import com.spotify.docker.client.DefaultDockerClient;
import com.spotify.docker.client.DockerClient;
import com.spotify.docker.client.LogStream;
@@ -27,6 +23,9 @@
import com.spotify.docker.client.messages.ContainerConfig;
import com.spotify.docker.client.messages.ExecCreation;
import com.spotify.docker.client.messages.ImageInfo;
+import java.io.File;
+import java.nio.file.Paths;
+import java.util.UUID;
import org.assertj.core.api.Assertions;
import org.gradle.testkit.runner.GradleRunner;
import org.junit.After;
@@ -35,97 +34,97 @@
public final class SparkDockerPluginSuite {
- private static final File TEST_PROJECT_DIR = Paths.get("src/test/resources/plugin-test-project").toFile();
+ private static final File TEST_PROJECT_DIR = Paths.get("src/test/resources/plugin-test-project").toFile();
- private String dockerTag;
+ private String dockerTag;
- @Before
- public void before() {
- dockerTag = UUID.randomUUID().toString().replaceAll("-", "");
- }
+ @Before
+ public void before() {
+ dockerTag = UUID.randomUUID().toString().replaceAll("-", "");
+ }
- @After
- public void after() throws Exception {
- try (DockerClient dockerClient = DefaultDockerClient.fromEnv().build()) {
- ImageInfo taggedImageInfo = dockerClient.inspectImage(
- String.format("docker.palantir.test/spark/spark-test-app:%s", dockerTag));
- dockerClient.removeImage(taggedImageInfo.id(), true, false);
+ @After
+ public void after() throws Exception {
+ try (DockerClient dockerClient = DefaultDockerClient.fromEnv().build()) {
+ ImageInfo taggedImageInfo = dockerClient.inspectImage(
+ String.format("docker.palantir.test/spark/spark-test-app:%s", dockerTag));
+ dockerClient.removeImage(taggedImageInfo.id(), true, false);
+ }
}
- }
- @Test
- public void testSetupProject() throws Exception {
- GradleRunner runner = GradleRunner.create()
- .withPluginClasspath()
- .withArguments(
- "clean",
- "sparkDockerTag",
- String.format("-Ddocker-tag=%s", dockerTag),
- "--stacktrace",
- "--info")
- .withProjectDir(TEST_PROJECT_DIR)
- .forwardOutput();
- runner.build();
+ @Test
+ public void testSetupProject() throws Exception {
+ GradleRunner runner = GradleRunner.create()
+ .withPluginClasspath()
+ .withArguments(
+ "clean",
+ "sparkDockerTag",
+ String.format("-Ddocker-tag=%s", dockerTag),
+ "--stacktrace",
+ "--info")
+ .withProjectDir(TEST_PROJECT_DIR)
+ .forwardOutput();
+ runner.build();
- try (DockerClient dockerClient = DefaultDockerClient.fromEnv().build()) {
- ImageInfo taggedImageInfo = dockerClient.inspectImage(
- String.format("docker.palantir.test/spark/spark-test-app:%s", dockerTag));
- Assertions.assertThat(taggedImageInfo).isNotNull();
- ContainerConfig containerConfig = ContainerConfig.builder()
- .entrypoint("bash")
- .cmd("-c", "while :; do sleep 1000; done")
- .image(taggedImageInfo.id())
- .build();
- String containerId = dockerClient.createContainer(containerConfig).id();
- try {
- dockerClient.startContainer(containerId);
- expectFilesInDir(
- dockerClient,
- containerId,
- "/opt/spark/jars",
- "guava-21.0.jar",
- "commons-io-2.4.jar",
- "plugin-test-project-1.0.jar");
- expectFilesInDir(
- dockerClient,
- containerId,
- "/opt/",
- "spark",
- "entrypoint.sh");
- expectFilesInDir(
- dockerClient,
- containerId,
- "/opt/spark/bin",
- "spark-submit");
- } finally {
- destroyContainer(dockerClient, containerId);
- }
+ try (DockerClient dockerClient = DefaultDockerClient.fromEnv().build()) {
+ ImageInfo taggedImageInfo = dockerClient.inspectImage(
+ String.format("docker.palantir.test/spark/spark-test-app:%s", dockerTag));
+ Assertions.assertThat(taggedImageInfo).isNotNull();
+ ContainerConfig containerConfig = ContainerConfig.builder()
+ .entrypoint("bash")
+ .cmd("-c", "while :; do sleep 1000; done")
+ .image(taggedImageInfo.id())
+ .build();
+ String containerId = dockerClient.createContainer(containerConfig).id();
+ try {
+ dockerClient.startContainer(containerId);
+ expectFilesInDir(
+ dockerClient,
+ containerId,
+ "/opt/spark/jars",
+ "guava-21.0.jar",
+ "commons-io-2.4.jar",
+ "plugin-test-project-1.0.jar");
+ expectFilesInDir(
+ dockerClient,
+ containerId,
+ "/opt/",
+ "spark",
+ "entrypoint.sh");
+ expectFilesInDir(
+ dockerClient,
+ containerId,
+ "/opt/spark/bin",
+ "spark-submit");
+ } finally {
+ destroyContainer(dockerClient, containerId);
+ }
+ }
}
- }
- private void expectFilesInDir(
- DockerClient dockerClient,
- String containerId,
- String path,
- String... expectedFiles) throws DockerException, InterruptedException {
- String[] listFilesCommand = new String[]{"bash", "-c", String.format("ls %s", path) };
- ExecCreation listFilesExec = dockerClient.execCreate(
- containerId,
- listFilesCommand,
- DockerClient.ExecCreateParam.attachStdout(),
- DockerClient.ExecCreateParam.attachStderr());
- try (LogStream listFilesLogs = dockerClient.execStart(listFilesExec.id())) {
- String output = listFilesLogs.readFully();
- Assertions.assertThat(output.split("\\s+")).contains(expectedFiles);
+ private void expectFilesInDir(
+ DockerClient dockerClient,
+ String containerId,
+ String path,
+ String... expectedFiles) throws DockerException, InterruptedException {
+ String[] listFilesCommand = new String[]{"bash", "-c", String.format("ls %s", path)};
+ ExecCreation listFilesExec = dockerClient.execCreate(
+ containerId,
+ listFilesCommand,
+ DockerClient.ExecCreateParam.attachStdout(),
+ DockerClient.ExecCreateParam.attachStderr());
+ try (LogStream listFilesLogs = dockerClient.execStart(listFilesExec.id())) {
+ String output = listFilesLogs.readFully();
+ Assertions.assertThat(output.split("\\s+")).contains(expectedFiles);
+ }
}
- }
- private void destroyContainer(DockerClient dockerClient, String containerId)
- throws DockerException, InterruptedException {
- try {
- dockerClient.killContainer(containerId);
- } finally {
- dockerClient.removeContainer(containerId);
+ private void destroyContainer(DockerClient dockerClient, String containerId)
+ throws DockerException, InterruptedException {
+ try {
+ dockerClient.killContainer(containerId);
+ } finally {
+ dockerClient.removeContainer(containerId);
+ }
}
- }
}
diff --git a/spark-docker-image-generator/src/test/resources/plugin-test-project/src/main/java/org/apache/spark/test/HelloWorld.java b/spark-docker-image-generator/src/test/resources/plugin-test-project/src/main/java/org/apache/spark/test/HelloWorld.java
index 8b7898fcac9d8..a47c4da4ff83d 100644
--- a/spark-docker-image-generator/src/test/resources/plugin-test-project/src/main/java/org/apache/spark/test/HelloWorld.java
+++ b/spark-docker-image-generator/src/test/resources/plugin-test-project/src/main/java/org/apache/spark/test/HelloWorld.java
@@ -24,10 +24,10 @@
public final class HelloWorld {
- private HelloWorld() { }
+ private HelloWorld() { }
- public static void main(String[] args) {
- List strs = ImmutableList.of("Hello", "World");
- System.out.println(Joiner.on(",").join(strs));
- }
+ public static void main(String[] args) {
+ List strs = ImmutableList.of("Hello", "World");
+ System.out.println(Joiner.on(",").join(strs));
+ }
}
diff --git a/versions.props b/versions.props
index 14c5000cc4b4e..93d671b558f24 100644
--- a/versions.props
+++ b/versions.props
@@ -17,7 +17,6 @@
org.apache.commons:commons-compress = 1.8.1
commons-io:commons-io = 2.4
-com.fasterxml.jackson:* = 2.8.0
com.spotify:docker-client = 8.11.2
org.assertj:assertj-core = 3.5.2
org.mockito:mockito-core = 2.2.11