Upgrade to Spark 3.3.3 (#2148) (#2153)

This PR upgrades our spark 3.x dependencies to version 3.3.3. (cherry picked from commit 35982db)
elastic · Nov 1, 2023 · 5269e8b · 5269e8b
1 parent cc36d02
commit 5269e8b
Show file tree

Hide file tree

Showing 24 changed files with 22 additions and 16 deletions.
diff --git a/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy
@@ -310,7 +310,11 @@ class BuildPlugin implements Plugin<Project>  {
                 resolve.eachDependency { DependencyResolveDetails details ->
                     // There are tons of slf4j-* variants. Search for all of them, and lock them down.
                     if (details.requested.name.contains("slf4j-")) {
-                        details.useVersion "1.7.6"
+                        // Some projects make use of the slf4j binding libraries that are from the implementation side,
+                        // so these must be left alone.
+                        if (details.requested.group.equals("org.apache.logging.log4j") == false) {
+                            details.useVersion "1.7.6"
+                        }
                     }
                     // Be careful with log4j version settings as they can be easily missed.
                     if (details.requested.name.contains("org.apache.logging.log4j") && details.requested.name.contains("log4j-")) {

diff --git a/...org/elasticsearch/hadoop/gradle/fixture/hadoop/services/SparkYarnServiceDescriptor.groovy b/...org/elasticsearch/hadoop/gradle/fixture/hadoop/services/SparkYarnServiceDescriptor.groovy
@@ -56,13 +56,13 @@ class SparkYarnServiceDescriptor implements ServiceDescriptor {
 
     @Override
     Version defaultVersion() {
-        return new Version(3, 2, 4)
+        return new Version(3, 3, 3)
     }
 
     String hadoopVersionCompatibility() {
         // The spark artifacts that interface with Hadoop have a hadoop version in their names.
         // This version is not always a version that Hadoop still distributes.
-        return "3.2"
+        return "3"
     }
 
     @Override

diff --git a/gradle.properties b/gradle.properties
@@ -28,7 +28,7 @@ spark13Version = 1.6.2
 spark20Version = 2.3.0
 spark22Version = 2.2.3
 spark24Version = 2.4.4
-spark30Version = 3.2.4
+spark30Version = 3.3.3
 
 # same as Spark's
 scala210Version = 2.10.7

diff --git a/licenses/spark-catalyst_2.12-3.2.4.jar.sha1 b/licenses/spark-catalyst_2.12-3.2.4.jar.sha1
diff --git a/licenses/spark-catalyst_2.12-3.3.3.jar.sha1 b/licenses/spark-catalyst_2.12-3.3.3.jar.sha1
@@ -0,0 +1 @@
+3f04550e73f020b96f83143beb238294d057e24b
diff --git a/licenses/spark-core_2.12-3.2.4.jar.sha1 b/licenses/spark-core_2.12-3.2.4.jar.sha1
diff --git a/licenses/spark-core_2.12-3.3.3.jar.sha1 b/licenses/spark-core_2.12-3.3.3.jar.sha1
@@ -0,0 +1 @@
+8c00f8f6caa081f6c047862c97caa254e0e79cc8
diff --git a/licenses/spark-sql_2.12-3.2.4.jar.sha1 b/licenses/spark-sql_2.12-3.2.4.jar.sha1
diff --git a/licenses/spark-sql_2.12-3.3.3.jar.sha1 b/licenses/spark-sql_2.12-3.3.3.jar.sha1
@@ -0,0 +1 @@
+be2a4df9c65624155eb299ffc677cabb58e51ac2
diff --git a/licenses/spark-streaming_2.12-3.2.4.jar.sha1 b/licenses/spark-streaming_2.12-3.2.4.jar.sha1
diff --git a/licenses/spark-streaming_2.12-3.3.3.jar.sha1 b/licenses/spark-streaming_2.12-3.3.3.jar.sha1
@@ -0,0 +1 @@
+fc09f62e7e5d00a9f7121c332a53a424108ff71f
diff --git a/licenses/spark-yarn_2.12-3.2.4.jar.sha1 b/licenses/spark-yarn_2.12-3.2.4.jar.sha1
diff --git a/licenses/spark-yarn_2.12-3.3.3.jar.sha1 b/licenses/spark-yarn_2.12-3.3.3.jar.sha1
@@ -0,0 +1 @@
+be516f2812fd7859c18dc2c216f4dd863a595ca1
diff --git a/spark/sql-30/licenses/spark-catalyst_2.12-3.2.4.jar.sha1 b/spark/sql-30/licenses/spark-catalyst_2.12-3.2.4.jar.sha1
diff --git a/spark/sql-30/licenses/spark-catalyst_2.12-3.3.3.jar.sha1 b/spark/sql-30/licenses/spark-catalyst_2.12-3.3.3.jar.sha1
@@ -0,0 +1 @@
+3f04550e73f020b96f83143beb238294d057e24b
diff --git a/spark/sql-30/licenses/spark-core_2.12-3.2.4.jar.sha1 b/spark/sql-30/licenses/spark-core_2.12-3.2.4.jar.sha1
diff --git a/spark/sql-30/licenses/spark-core_2.12-3.3.3.jar.sha1 b/spark/sql-30/licenses/spark-core_2.12-3.3.3.jar.sha1
@@ -0,0 +1 @@
+8c00f8f6caa081f6c047862c97caa254e0e79cc8
diff --git a/spark/sql-30/licenses/spark-sql_2.12-3.2.4.jar.sha1 b/spark/sql-30/licenses/spark-sql_2.12-3.2.4.jar.sha1
diff --git a/spark/sql-30/licenses/spark-sql_2.12-3.3.3.jar.sha1 b/spark/sql-30/licenses/spark-sql_2.12-3.3.3.jar.sha1
@@ -0,0 +1 @@
+be2a4df9c65624155eb299ffc677cabb58e51ac2
diff --git a/spark/sql-30/licenses/spark-streaming_2.12-3.2.4.jar.sha1 b/spark/sql-30/licenses/spark-streaming_2.12-3.2.4.jar.sha1
diff --git a/spark/sql-30/licenses/spark-streaming_2.12-3.3.3.jar.sha1 b/spark/sql-30/licenses/spark-streaming_2.12-3.3.3.jar.sha1
@@ -0,0 +1 @@
+fc09f62e7e5d00a9f7121c332a53a424108ff71f
diff --git a/spark/sql-30/licenses/spark-yarn_2.12-3.2.4.jar.sha1 b/spark/sql-30/licenses/spark-yarn_2.12-3.2.4.jar.sha1
diff --git a/spark/sql-30/licenses/spark-yarn_2.12-3.3.3.jar.sha1 b/spark/sql-30/licenses/spark-yarn_2.12-3.3.3.jar.sha1
@@ -0,0 +1 @@
+be516f2812fd7859c18dc2c216f4dd863a595ca1
diff --git a/...ql-30/src/test/scala/org/elasticsearch/spark/sql/RowSerializationEventConverterTest.scala b/...ql-30/src/test/scala/org/elasticsearch/spark/sql/RowSerializationEventConverterTest.scala
@@ -50,8 +50,10 @@ class RowSerializationEventConverterTest {
     val rawEvent = eventConverter.getRawEvent(iaeFailure)
 
     // Scala 2.13 changed what toString() looks like, so can't do an exact match here:
-    assertTrue(rawEvent.contains("(StructField(field1,StringType,true), " +
-      "StructField(field2,StringType,true), StructField(field3,StringType,true)),[value1,value2,value3])"))
+    assertTrue(rawEvent.contains("(StructField(field1,StringType,true),"))
+    assertTrue(rawEvent.contains("StructField(field2,StringType,true),"))
+    assertTrue(rawEvent.contains("StructField(field3,StringType,true)),"))
+    assertTrue(rawEvent.contains("[value1,value2,value3])"))
     val timestamp = eventConverter.getTimestamp(iaeFailure)
     assertTrue(StringUtils.hasText(timestamp))
     assertTrue(DateUtils.parseDate(timestamp).getTime.getTime > 1L)