diff --git a/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy index f4fbf6196..7536694ab 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy @@ -310,7 +310,11 @@ class BuildPlugin implements Plugin { resolve.eachDependency { DependencyResolveDetails details -> // There are tons of slf4j-* variants. Search for all of them, and lock them down. if (details.requested.name.contains("slf4j-")) { - details.useVersion "1.7.6" + // Some projects make use of the slf4j binding libraries that are from the implementation side, + // so these must be left alone. + if (details.requested.group.equals("org.apache.logging.log4j") == false) { + details.useVersion "1.7.6" + } } // Be careful with log4j version settings as they can be easily missed. if (details.requested.name.contains("org.apache.logging.log4j") && details.requested.name.contains("log4j-")) { diff --git a/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/services/SparkYarnServiceDescriptor.groovy b/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/services/SparkYarnServiceDescriptor.groovy index 9f4975d09..c90a9d2ce 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/services/SparkYarnServiceDescriptor.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/fixture/hadoop/services/SparkYarnServiceDescriptor.groovy @@ -56,13 +56,13 @@ class SparkYarnServiceDescriptor implements ServiceDescriptor { @Override Version defaultVersion() { - return new Version(3, 2, 4) + return new Version(3, 3, 3) } String hadoopVersionCompatibility() { // The spark artifacts that interface with Hadoop have a hadoop version in their names. // This version is not always a version that Hadoop still distributes. - return "3.2" + return "3" } @Override diff --git a/gradle.properties b/gradle.properties index 1cb44bd27..e935ec514 100644 --- a/gradle.properties +++ b/gradle.properties @@ -28,7 +28,7 @@ spark13Version = 1.6.2 spark20Version = 2.3.0 spark22Version = 2.2.3 spark24Version = 2.4.4 -spark30Version = 3.2.4 +spark30Version = 3.3.3 # same as Spark's scala210Version = 2.10.7 diff --git a/licenses/spark-catalyst_2.12-3.2.4.jar.sha1 b/licenses/spark-catalyst_2.12-3.2.4.jar.sha1 deleted file mode 100644 index fa2da1917..000000000 --- a/licenses/spark-catalyst_2.12-3.2.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bfc16bcbcc2d6e65c61b1f3214c56e7ea998de90 \ No newline at end of file diff --git a/licenses/spark-catalyst_2.12-3.3.3.jar.sha1 b/licenses/spark-catalyst_2.12-3.3.3.jar.sha1 new file mode 100644 index 000000000..0f2b6c73f --- /dev/null +++ b/licenses/spark-catalyst_2.12-3.3.3.jar.sha1 @@ -0,0 +1 @@ +3f04550e73f020b96f83143beb238294d057e24b \ No newline at end of file diff --git a/licenses/spark-core_2.12-3.2.4.jar.sha1 b/licenses/spark-core_2.12-3.2.4.jar.sha1 deleted file mode 100644 index bb969b357..000000000 --- a/licenses/spark-core_2.12-3.2.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c61b811e273c04bb53931ab02d2648248033c445 \ No newline at end of file diff --git a/licenses/spark-core_2.12-3.3.3.jar.sha1 b/licenses/spark-core_2.12-3.3.3.jar.sha1 new file mode 100644 index 000000000..e1faa95f2 --- /dev/null +++ b/licenses/spark-core_2.12-3.3.3.jar.sha1 @@ -0,0 +1 @@ +8c00f8f6caa081f6c047862c97caa254e0e79cc8 \ No newline at end of file diff --git a/licenses/spark-sql_2.12-3.2.4.jar.sha1 b/licenses/spark-sql_2.12-3.2.4.jar.sha1 deleted file mode 100644 index 0a4d73358..000000000 --- a/licenses/spark-sql_2.12-3.2.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9bbe7a5fd0478bb7f3a2eb2e5dcb004f4ed70305 \ No newline at end of file diff --git a/licenses/spark-sql_2.12-3.3.3.jar.sha1 b/licenses/spark-sql_2.12-3.3.3.jar.sha1 new file mode 100644 index 000000000..4677c9466 --- /dev/null +++ b/licenses/spark-sql_2.12-3.3.3.jar.sha1 @@ -0,0 +1 @@ +be2a4df9c65624155eb299ffc677cabb58e51ac2 \ No newline at end of file diff --git a/licenses/spark-streaming_2.12-3.2.4.jar.sha1 b/licenses/spark-streaming_2.12-3.2.4.jar.sha1 deleted file mode 100644 index 39d1f789d..000000000 --- a/licenses/spark-streaming_2.12-3.2.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ce150a20b26405f5b209205a4b64bc570d5a551f \ No newline at end of file diff --git a/licenses/spark-streaming_2.12-3.3.3.jar.sha1 b/licenses/spark-streaming_2.12-3.3.3.jar.sha1 new file mode 100644 index 000000000..d8e53204c --- /dev/null +++ b/licenses/spark-streaming_2.12-3.3.3.jar.sha1 @@ -0,0 +1 @@ +fc09f62e7e5d00a9f7121c332a53a424108ff71f \ No newline at end of file diff --git a/licenses/spark-yarn_2.12-3.2.4.jar.sha1 b/licenses/spark-yarn_2.12-3.2.4.jar.sha1 deleted file mode 100644 index bb200a040..000000000 --- a/licenses/spark-yarn_2.12-3.2.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -48f903127f7c2b69daaf30200d19d4d750e5af4a \ No newline at end of file diff --git a/licenses/spark-yarn_2.12-3.3.3.jar.sha1 b/licenses/spark-yarn_2.12-3.3.3.jar.sha1 new file mode 100644 index 000000000..4600c988e --- /dev/null +++ b/licenses/spark-yarn_2.12-3.3.3.jar.sha1 @@ -0,0 +1 @@ +be516f2812fd7859c18dc2c216f4dd863a595ca1 \ No newline at end of file diff --git a/spark/sql-30/licenses/spark-catalyst_2.12-3.2.4.jar.sha1 b/spark/sql-30/licenses/spark-catalyst_2.12-3.2.4.jar.sha1 deleted file mode 100644 index fa2da1917..000000000 --- a/spark/sql-30/licenses/spark-catalyst_2.12-3.2.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -bfc16bcbcc2d6e65c61b1f3214c56e7ea998de90 \ No newline at end of file diff --git a/spark/sql-30/licenses/spark-catalyst_2.12-3.3.3.jar.sha1 b/spark/sql-30/licenses/spark-catalyst_2.12-3.3.3.jar.sha1 new file mode 100644 index 000000000..0f2b6c73f --- /dev/null +++ b/spark/sql-30/licenses/spark-catalyst_2.12-3.3.3.jar.sha1 @@ -0,0 +1 @@ +3f04550e73f020b96f83143beb238294d057e24b \ No newline at end of file diff --git a/spark/sql-30/licenses/spark-core_2.12-3.2.4.jar.sha1 b/spark/sql-30/licenses/spark-core_2.12-3.2.4.jar.sha1 deleted file mode 100644 index bb969b357..000000000 --- a/spark/sql-30/licenses/spark-core_2.12-3.2.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c61b811e273c04bb53931ab02d2648248033c445 \ No newline at end of file diff --git a/spark/sql-30/licenses/spark-core_2.12-3.3.3.jar.sha1 b/spark/sql-30/licenses/spark-core_2.12-3.3.3.jar.sha1 new file mode 100644 index 000000000..e1faa95f2 --- /dev/null +++ b/spark/sql-30/licenses/spark-core_2.12-3.3.3.jar.sha1 @@ -0,0 +1 @@ +8c00f8f6caa081f6c047862c97caa254e0e79cc8 \ No newline at end of file diff --git a/spark/sql-30/licenses/spark-sql_2.12-3.2.4.jar.sha1 b/spark/sql-30/licenses/spark-sql_2.12-3.2.4.jar.sha1 deleted file mode 100644 index 0a4d73358..000000000 --- a/spark/sql-30/licenses/spark-sql_2.12-3.2.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9bbe7a5fd0478bb7f3a2eb2e5dcb004f4ed70305 \ No newline at end of file diff --git a/spark/sql-30/licenses/spark-sql_2.12-3.3.3.jar.sha1 b/spark/sql-30/licenses/spark-sql_2.12-3.3.3.jar.sha1 new file mode 100644 index 000000000..4677c9466 --- /dev/null +++ b/spark/sql-30/licenses/spark-sql_2.12-3.3.3.jar.sha1 @@ -0,0 +1 @@ +be2a4df9c65624155eb299ffc677cabb58e51ac2 \ No newline at end of file diff --git a/spark/sql-30/licenses/spark-streaming_2.12-3.2.4.jar.sha1 b/spark/sql-30/licenses/spark-streaming_2.12-3.2.4.jar.sha1 deleted file mode 100644 index 39d1f789d..000000000 --- a/spark/sql-30/licenses/spark-streaming_2.12-3.2.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ce150a20b26405f5b209205a4b64bc570d5a551f \ No newline at end of file diff --git a/spark/sql-30/licenses/spark-streaming_2.12-3.3.3.jar.sha1 b/spark/sql-30/licenses/spark-streaming_2.12-3.3.3.jar.sha1 new file mode 100644 index 000000000..d8e53204c --- /dev/null +++ b/spark/sql-30/licenses/spark-streaming_2.12-3.3.3.jar.sha1 @@ -0,0 +1 @@ +fc09f62e7e5d00a9f7121c332a53a424108ff71f \ No newline at end of file diff --git a/spark/sql-30/licenses/spark-yarn_2.12-3.2.4.jar.sha1 b/spark/sql-30/licenses/spark-yarn_2.12-3.2.4.jar.sha1 deleted file mode 100644 index bb200a040..000000000 --- a/spark/sql-30/licenses/spark-yarn_2.12-3.2.4.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -48f903127f7c2b69daaf30200d19d4d750e5af4a \ No newline at end of file diff --git a/spark/sql-30/licenses/spark-yarn_2.12-3.3.3.jar.sha1 b/spark/sql-30/licenses/spark-yarn_2.12-3.3.3.jar.sha1 new file mode 100644 index 000000000..4600c988e --- /dev/null +++ b/spark/sql-30/licenses/spark-yarn_2.12-3.3.3.jar.sha1 @@ -0,0 +1 @@ +be516f2812fd7859c18dc2c216f4dd863a595ca1 \ No newline at end of file diff --git a/spark/sql-30/src/test/scala/org/elasticsearch/spark/sql/RowSerializationEventConverterTest.scala b/spark/sql-30/src/test/scala/org/elasticsearch/spark/sql/RowSerializationEventConverterTest.scala index 74138e27c..41fad7be5 100644 --- a/spark/sql-30/src/test/scala/org/elasticsearch/spark/sql/RowSerializationEventConverterTest.scala +++ b/spark/sql-30/src/test/scala/org/elasticsearch/spark/sql/RowSerializationEventConverterTest.scala @@ -50,8 +50,10 @@ class RowSerializationEventConverterTest { val rawEvent = eventConverter.getRawEvent(iaeFailure) // Scala 2.13 changed what toString() looks like, so can't do an exact match here: - assertTrue(rawEvent.contains("(StructField(field1,StringType,true), " + - "StructField(field2,StringType,true), StructField(field3,StringType,true)),[value1,value2,value3])")) + assertTrue(rawEvent.contains("(StructField(field1,StringType,true),")) + assertTrue(rawEvent.contains("StructField(field2,StringType,true),")) + assertTrue(rawEvent.contains("StructField(field3,StringType,true)),")) + assertTrue(rawEvent.contains("[value1,value2,value3])")) val timestamp = eventConverter.getTimestamp(iaeFailure) assertTrue(StringUtils.hasText(timestamp)) assertTrue(DateUtils.parseDate(timestamp).getTime.getTime > 1L)