From a0b5c60ba2098e0b3ed1df56501cfe91bf6893bb Mon Sep 17 00:00:00 2001 From: Lin Zhou Date: Thu, 20 Jul 2023 17:28:46 -0700 Subject: [PATCH 01/23] Update Python connector version to --- python/delta_sharing/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/delta_sharing/version.py b/python/delta_sharing/version.py index dfeecc4de..2b3708b64 100644 --- a/python/delta_sharing/version.py +++ b/python/delta_sharing/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.7.1" +__version__ = "" From 6540d33f70dfa9be6939db43921c2506470ce124 Mon Sep 17 00:00:00 2001 From: Lin Zhou Date: Thu, 20 Jul 2023 17:29:02 -0700 Subject: [PATCH 02/23] Update Python connector version to 0.8.0 --- python/delta_sharing/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/delta_sharing/version.py b/python/delta_sharing/version.py index 2b3708b64..d9149e7fe 100644 --- a/python/delta_sharing/version.py +++ b/python/delta_sharing/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "" +__version__ = "0.8.0" From 906f55316c3d11b4d9f0b73e30d16959e280123f Mon Sep 17 00:00:00 2001 From: Lin Zhou Date: Thu, 20 Jul 2023 17:29:28 -0700 Subject: [PATCH 03/23] update to scala2.13 and spark 3.3.2 --- build.sbt | 11 ++++++++--- .../sharing/spark/DeltaSharingClient.scala | 18 +++++++++--------- .../sharing/spark/DeltaSharingDataSource.scala | 2 +- .../sharing/spark/DeltaSharingSource.scala | 6 +++--- .../sharing/spark/RemoteDeltaCDFRelation.scala | 2 +- .../io/delta/sharing/spark/TestUtils.scala | 4 +--- 6 files changed, 23 insertions(+), 20 deletions(-) diff --git a/build.sbt b/build.sbt index 68c395908..edac0ab7d 100644 --- a/build.sbt +++ b/build.sbt @@ -18,11 +18,12 @@ import sbt.ExclusionRule ThisBuild / parallelExecution := false -val sparkVersion = "3.1.1" +val sparkVersion = "3.3.2" +val scala212 = "2.12.10" +val scala213 = "2.13.5" lazy val commonSettings = Seq( organization := "io.delta", - scalaVersion := "2.12.10", fork := true, javacOptions ++= Seq("-source", "1.8", "-target", "1.8"), scalacOptions += "-target:jvm-1.8", @@ -43,10 +44,13 @@ lazy val root = (project in file(".")).aggregate(spark, server) lazy val spark = (project in file("spark")) settings( name := "delta-sharing-spark", + crossScalaVersions := Seq(scala212, scala213), commonSettings, scalaStyleSettings, releaseSettings, libraryDependencies ++= Seq( + "org.apache.httpcomponents" % "httpclient" % "4.5.13", + "org.codehaus.jackson" % "jackson-mapper-asl" % "1.9.13", "org.apache.spark" %% "spark-sql" % sparkVersion % "provided", "org.apache.spark" %% "spark-catalyst" % sparkVersion % "test" classifier "tests", "org.apache.spark" %% "spark-core" % sparkVersion % "test" classifier "tests", @@ -68,6 +72,7 @@ lazy val spark = (project in file("spark")) settings( lazy val server = (project in file("server")) enablePlugins(JavaAppPackaging) settings( name := "delta-sharing-server", + scalaVersion := scala212, commonSettings, scalaStyleSettings, releaseSettings, @@ -246,4 +251,4 @@ releaseProcess := Seq[ReleaseStep]( publishArtifacts, setNextVersion, commitNextVersion -) +) \ No newline at end of file diff --git a/spark/src/main/scala/io/delta/sharing/spark/DeltaSharingClient.scala b/spark/src/main/scala/io/delta/sharing/spark/DeltaSharingClient.scala index 775171d51..43012ea8c 100644 --- a/spark/src/main/scala/io/delta/sharing/spark/DeltaSharingClient.scala +++ b/spark/src/main/scala/io/delta/sharing/spark/DeltaSharingClient.scala @@ -154,7 +154,7 @@ private[spark] class DeltaSharingRestClient( shares ++= response.items } } - shares + shares.toSeq } private def listAllTablesInShare(share: Share): Seq[Table] = { @@ -174,7 +174,7 @@ private[spark] class DeltaSharingRestClient( tables ++= response.items } } - tables + tables.toSeq } override def getForStreaming(): Boolean = forStreaming @@ -282,9 +282,9 @@ private[spark] class DeltaSharingRestClient( version, protocol, metadata, - addFiles = addFiles, - removeFiles = removeFiles, - additionalMetadatas = additionalMetadatas + addFiles = addFiles.toSeq, + removeFiles = removeFiles.toSeq, + additionalMetadatas = additionalMetadatas.toSeq ) } @@ -319,10 +319,10 @@ private[spark] class DeltaSharingRestClient( version, protocol, metadata, - addFiles = addFiles, - cdfFiles = cdfFiles, - removeFiles = removeFiles, - additionalMetadatas = additionalMetadatas + addFiles = addFiles.toSeq, + cdfFiles = cdfFiles.toSeq, + removeFiles = removeFiles.toSeq, + additionalMetadatas = additionalMetadatas.toSeq ) } diff --git a/spark/src/main/scala/io/delta/sharing/spark/DeltaSharingDataSource.scala b/spark/src/main/scala/io/delta/sharing/spark/DeltaSharingDataSource.scala index 6bf620a94..fd02c5e11 100644 --- a/spark/src/main/scala/io/delta/sharing/spark/DeltaSharingDataSource.scala +++ b/spark/src/main/scala/io/delta/sharing/spark/DeltaSharingDataSource.scala @@ -98,7 +98,7 @@ private[sharing] class DeltaSharingDataSource DeltaSharingSource(SparkSession.active, deltaLog, options) } - override def shortName: String = "deltaSharing" + override def shortName(): String = "deltaSharing" } private[sharing] object DeltaSharingDataSource { diff --git a/spark/src/main/scala/io/delta/sharing/spark/DeltaSharingSource.scala b/spark/src/main/scala/io/delta/sharing/spark/DeltaSharingSource.scala index a77f967b4..92b3e83d5 100644 --- a/spark/src/main/scala/io/delta/sharing/spark/DeltaSharingSource.scala +++ b/spark/src/main/scala/io/delta/sharing/spark/DeltaSharingSource.scala @@ -801,9 +801,9 @@ case class DeltaSharingSource( DeltaSharingCDFReader.changesToDF( new RemoteDeltaFileIndexParams(spark, initSnapshot, deltaLog.client.getProfileProvider), schema.fields.map(f => f.name), - addFiles, - cdfFiles, - removeFiles, + addFiles.toSeq, + cdfFiles.toSeq, + removeFiles.toSeq, schema, isStreaming = true, latestRefreshFunc, diff --git a/spark/src/main/scala/io/delta/sharing/spark/RemoteDeltaCDFRelation.scala b/spark/src/main/scala/io/delta/sharing/spark/RemoteDeltaCDFRelation.scala index 399097bed..e1e96893e 100644 --- a/spark/src/main/scala/io/delta/sharing/spark/RemoteDeltaCDFRelation.scala +++ b/spark/src/main/scala/io/delta/sharing/spark/RemoteDeltaCDFRelation.scala @@ -103,7 +103,7 @@ object DeltaSharingCDFReader { CachedTableManager.INSTANCE.register( params.path.toString, getIdToUrl(addFiles, cdfFiles, removeFiles), - refs, + refs.toSeq, params.profileProvider, refresher, if (expirationTimestamp.isDefined) { diff --git a/spark/src/test/scala/io/delta/sharing/spark/TestUtils.scala b/spark/src/test/scala/io/delta/sharing/spark/TestUtils.scala index 80f08d783..93bf55ef0 100644 --- a/spark/src/test/scala/io/delta/sharing/spark/TestUtils.scala +++ b/spark/src/test/scala/io/delta/sharing/spark/TestUtils.scala @@ -43,9 +43,7 @@ object TestUtils { } def sqlDate(date: String): java.sql.Date = { - toJavaDate(stringToDate( - UTF8String.fromString(date), - getZoneId(SQLConf.get.sessionLocalTimeZone)).get) + toJavaDate(stringToDate(UTF8String.fromString(date)).get) } def sqlTimestamp(timestamp: String): java.sql.Timestamp = { From 6e4cbf17511c90512a6cc2048ede6259f41401d8 Mon Sep 17 00:00:00 2001 From: Lin Zhou Date: Thu, 20 Jul 2023 17:30:20 -0700 Subject: [PATCH 04/23] Update Python connector version to --- python/delta_sharing/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/delta_sharing/version.py b/python/delta_sharing/version.py index d9149e7fe..2b3708b64 100644 --- a/python/delta_sharing/version.py +++ b/python/delta_sharing/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "0.8.0" +__version__ = "" From b3d49facd594adbf241b3880ec9c2ad943d4fb78 Mon Sep 17 00:00:00 2001 From: Lin Zhou Date: Thu, 20 Jul 2023 17:30:56 -0700 Subject: [PATCH 05/23] Update Python connector version to 0.8.0 --- python/delta_sharing/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/delta_sharing/version.py b/python/delta_sharing/version.py index 2b3708b64..d9149e7fe 100644 --- a/python/delta_sharing/version.py +++ b/python/delta_sharing/version.py @@ -14,4 +14,4 @@ # limitations under the License. # -__version__ = "" +__version__ = "0.8.0" From 2171c48e30cf20ad1cc70cec253b5ba29dbc8e58 Mon Sep 17 00:00:00 2001 From: Lin Zhou Date: Thu, 20 Jul 2023 17:31:06 -0700 Subject: [PATCH 06/23] Setting version to 0.8.0 --- version.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.sbt b/version.sbt index dd8f3d717..c3945bb4f 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -version in ThisBuild := "0.7.2-SNAPSHOT" +version in ThisBuild := "0.8.0" From abc9b8370fe204648e0273d261175cccaf3d94f4 Mon Sep 17 00:00:00 2001 From: Lin Zhou Date: Thu, 20 Jul 2023 17:35:41 -0700 Subject: [PATCH 07/23] Update Python connector version to 0.8.0 From e1ff65659749aacc8630eabc8efcd5f5fa2dfbae Mon Sep 17 00:00:00 2001 From: Lin Zhou Date: Thu, 20 Jul 2023 17:37:36 -0700 Subject: [PATCH 08/23] release change --- dev/release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/release.sh b/dev/release.sh index fe0a19af7..8a27633ce 100755 --- a/dev/release.sh +++ b/dev/release.sh @@ -26,7 +26,7 @@ git add python/delta_sharing/version.py # Use --allow-empty so that we can re-run this script even if the Python connector version has been updated git commit -m "Update Python connector version to $VERSION" --allow-empty -build/sbt "release skip-tests" +build/sbt "+ release skip-tests" # Switch to the release commit git checkout v$VERSION From e96584e78486564324c6f1c39c3ee039a335c59e Mon Sep 17 00:00:00 2001 From: Lin Zhou Date: Thu, 20 Jul 2023 17:37:48 -0700 Subject: [PATCH 09/23] Update Python connector version to 0.8.0 From 7a6b672cb82bff595d7c10b41194773d70c98a9c Mon Sep 17 00:00:00 2001 From: Lin Zhou Date: Thu, 20 Jul 2023 17:38:31 -0700 Subject: [PATCH 10/23] Setting version to 0.8.1-SNAPSHOT --- version.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.sbt b/version.sbt index c3945bb4f..e74ced496 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -version in ThisBuild := "0.8.0" +version in ThisBuild := "0.8.1-SNAPSHOT" From 3ee229509f81147df0505c0cf71de79655b45fe5 Mon Sep 17 00:00:00 2001 From: Lin Zhou Date: Fri, 21 Jul 2023 14:15:32 -0700 Subject: [PATCH 11/23] fix test --- .../sharing/spark/DeltaSharingFileProfileProviderSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/test/scala/io/delta/sharing/spark/DeltaSharingFileProfileProviderSuite.scala b/spark/src/test/scala/io/delta/sharing/spark/DeltaSharingFileProfileProviderSuite.scala index 8a975df4d..a5896f66a 100644 --- a/spark/src/test/scala/io/delta/sharing/spark/DeltaSharingFileProfileProviderSuite.scala +++ b/spark/src/test/scala/io/delta/sharing/spark/DeltaSharingFileProfileProviderSuite.scala @@ -71,7 +71,7 @@ class DeltaSharingFileProfileProviderSuite extends SparkFunSuite { } test("version is missing") { - val e = intercept[IllegalArgumentException] { + val e = intercept[Exception] { testProfile( """{ | "endpoint": "foo", From f58d05fc919c48dfe5753f39447c0eafcb5d6f97 Mon Sep 17 00:00:00 2001 From: Lin Zhou Date: Fri, 21 Jul 2023 14:29:39 -0700 Subject: [PATCH 12/23] fix test --- .../io/delta/sharing/spark/DeltaSharingProfileProvider.scala | 2 +- .../sharing/spark/DeltaSharingFileProfileProviderSuite.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spark/src/main/scala/io/delta/sharing/spark/DeltaSharingProfileProvider.scala b/spark/src/main/scala/io/delta/sharing/spark/DeltaSharingProfileProvider.scala index ae76b4fa0..f6c603f4d 100644 --- a/spark/src/main/scala/io/delta/sharing/spark/DeltaSharingProfileProvider.scala +++ b/spark/src/main/scala/io/delta/sharing/spark/DeltaSharingProfileProvider.scala @@ -25,7 +25,7 @@ import org.apache.hadoop.fs.Path import io.delta.sharing.spark.util.JsonUtils case class DeltaSharingProfile( - shareCredentialsVersion: Option[Int] = Some(DeltaSharingProfile.CURRENT), + shareCredentialsVersion: Option[Int], endpoint: String = null, bearerToken: String = null, expirationTime: String = null) diff --git a/spark/src/test/scala/io/delta/sharing/spark/DeltaSharingFileProfileProviderSuite.scala b/spark/src/test/scala/io/delta/sharing/spark/DeltaSharingFileProfileProviderSuite.scala index a5896f66a..e94c6a7f5 100644 --- a/spark/src/test/scala/io/delta/sharing/spark/DeltaSharingFileProfileProviderSuite.scala +++ b/spark/src/test/scala/io/delta/sharing/spark/DeltaSharingFileProfileProviderSuite.scala @@ -70,8 +70,8 @@ class DeltaSharingFileProfileProviderSuite extends SparkFunSuite { ) } - test("version is missing") { - val e = intercept[Exception] { + test("shareCredentialsVersion is missing") { + val e = intercept[IllegalArgumentException] { testProfile( """{ | "endpoint": "foo", From 77cbc2a62dd782ef185290fd0ac0e46468885e36 Mon Sep 17 00:00:00 2001 From: Lin Zhou Date: Fri, 21 Jul 2023 14:34:28 -0700 Subject: [PATCH 13/23] use 2.13.11 --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index edac0ab7d..97531a246 100644 --- a/build.sbt +++ b/build.sbt @@ -20,7 +20,7 @@ ThisBuild / parallelExecution := false val sparkVersion = "3.3.2" val scala212 = "2.12.10" -val scala213 = "2.13.5" +val scala213 = "2.13.11" lazy val commonSettings = Seq( organization := "io.delta", From 2eb1ce04296bd113374730807f6cd6030f38dcc7 Mon Sep 17 00:00:00 2001 From: Lin Zhou Date: Fri, 21 Jul 2023 14:36:42 -0700 Subject: [PATCH 14/23] add more developers --- build.sbt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/build.sbt b/build.sbt index 97531a246..a436eaafc 100644 --- a/build.sbt +++ b/build.sbt @@ -232,6 +232,16 @@ lazy val releaseSettings = Seq( Shixiong Zhu https://github.com/zsxwing + + linzhou-db + Lin Zhou + https://github.com/linzhou-db + + + chakankardb + Abhijit Chakankar + https://github.com/chakankardb + ) From 9b48e2fa7ce65af03b36fdcf16c490c67635b43b Mon Sep 17 00:00:00 2001 From: Lin Zhou Date: Fri, 21 Jul 2023 15:00:01 -0700 Subject: [PATCH 15/23] add for cross release --- build.sbt | 3 +++ dev/release.sh | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index a436eaafc..762f21812 100644 --- a/build.sbt +++ b/build.sbt @@ -250,6 +250,8 @@ publishArtifact := false // Don't release the root project publish := {} publishTo := Some("snapshots" at "https://oss.sonatype.org/content/repositories/snapshots") releaseCrossBuild := false +// crossScalaVersions must be set to Nil on the root project +crossScalaVersions := Nil releaseProcess := Seq[ReleaseStep]( checkSnapshotDependencies, inquireVersions, @@ -258,6 +260,7 @@ releaseProcess := Seq[ReleaseStep]( setReleaseVersion, commitReleaseVersion, tagRelease, + releaseStepCommandAndRemaining("+publishSigned"), publishArtifacts, setNextVersion, commitNextVersion diff --git a/dev/release.sh b/dev/release.sh index 8a27633ce..ed2bba92b 100755 --- a/dev/release.sh +++ b/dev/release.sh @@ -26,7 +26,7 @@ git add python/delta_sharing/version.py # Use --allow-empty so that we can re-run this script even if the Python connector version has been updated git commit -m "Update Python connector version to $VERSION" --allow-empty -build/sbt "+ release skip-tests" +build/sbt " release skip-tests" # Switch to the release commit git checkout v$VERSION From 52cb89c666fec9ffe26d902a2ac854ee7fd0855a Mon Sep 17 00:00:00 2001 From: Lin Zhou Date: Fri, 21 Jul 2023 15:01:36 -0700 Subject: [PATCH 16/23] lint --- build.sbt | 2 +- dev/release.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index 762f21812..89058e6c5 100644 --- a/build.sbt +++ b/build.sbt @@ -264,4 +264,4 @@ releaseProcess := Seq[ReleaseStep]( publishArtifacts, setNextVersion, commitNextVersion -) \ No newline at end of file +) diff --git a/dev/release.sh b/dev/release.sh index ed2bba92b..fe0a19af7 100755 --- a/dev/release.sh +++ b/dev/release.sh @@ -26,7 +26,7 @@ git add python/delta_sharing/version.py # Use --allow-empty so that we can re-run this script even if the Python connector version has been updated git commit -m "Update Python connector version to $VERSION" --allow-empty -build/sbt " release skip-tests" +build/sbt "release skip-tests" # Switch to the release commit git checkout v$VERSION From 3a670b85c977c3ffb3bab92034d79600eb40ba37 Mon Sep 17 00:00:00 2001 From: Lin Zhou Date: Fri, 21 Jul 2023 15:03:19 -0700 Subject: [PATCH 17/23] Update Python connector version to 0.8.0 From 6535ade777a3a6899bf51567545aa2b29cad792a Mon Sep 17 00:00:00 2001 From: Lin Zhou Date: Fri, 21 Jul 2023 15:03:29 -0700 Subject: [PATCH 18/23] Setting version to 0.8.0 --- version.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.sbt b/version.sbt index e74ced496..c3945bb4f 100644 --- a/version.sbt +++ b/version.sbt @@ -1 +1 @@ -version in ThisBuild := "0.8.1-SNAPSHOT" +version in ThisBuild := "0.8.0" From c5e9180d5493f6d1c6c5b3f668de4fcbae1af643 Mon Sep 17 00:00:00 2001 From: Lin Zhou Date: Fri, 21 Jul 2023 15:06:21 -0700 Subject: [PATCH 19/23] Update Python connector version to 0.8.0 From 9db2399fb03f07a71cd82121daeb4917ab90e46c Mon Sep 17 00:00:00 2001 From: Lin Zhou Date: Fri, 21 Jul 2023 15:08:09 -0700 Subject: [PATCH 20/23] Update Python connector version to 0.8.0 From 8f035bee67af03daa2591613b2d440e30c343461 Mon Sep 17 00:00:00 2001 From: Lin Zhou Date: Fri, 21 Jul 2023 15:13:57 -0700 Subject: [PATCH 21/23] Update Python connector version to 0.8.0 From a00d4dca054b47fe87ee336bd354aaa4305e5764 Mon Sep 17 00:00:00 2001 From: Lin Zhou Date: Fri, 21 Jul 2023 15:19:30 -0700 Subject: [PATCH 22/23] Update Python connector version to 0.8.0 From 5269a41e48b62986f995732fd1566c5dcb517115 Mon Sep 17 00:00:00 2001 From: Lin Zhou Date: Fri, 21 Jul 2023 15:19:56 -0700 Subject: [PATCH 23/23] update releaseSettings --- build.sbt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index 89058e6c5..a8122458f 100644 --- a/build.sbt +++ b/build.sbt @@ -184,6 +184,8 @@ import ReleaseTransformations._ lazy val releaseSettings = Seq( publishMavenStyle := true, + publishArtifact := true, + Test / publishArtifact := false, publishTo := { val nexus = "https://oss.sonatype.org/" @@ -194,6 +196,7 @@ lazy val releaseSettings = Seq( } }, + releasePublishArtifactsAction := PgpKeys.publishSigned.value, releaseCrossBuild := true, @@ -261,7 +264,6 @@ releaseProcess := Seq[ReleaseStep]( commitReleaseVersion, tagRelease, releaseStepCommandAndRemaining("+publishSigned"), - publishArtifacts, setNextVersion, commitNextVersion )