Merge branch 'master' into feat/split-elastic

Signed-off-by: David Sloan <[email protected]>
lensesio · Apr 2, 2024 · ca3a62e · ca3a62e
2 parents 5832ac9 + bce4a33
commit ca3a62e
Show file tree

Hide file tree

Showing 46 changed files with 682 additions and 290 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -22,9 +22,9 @@ jobs:
       fun_matrix: ${{ steps.read-mods.outputs.fun-matrix }}
       dep_check_matrix: ${{ steps.read-mods.outputs.dep-check-matrix }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '17'
           distribution: 'temurin'
@@ -51,9 +51,9 @@ jobs:
     timeout-minutes: 5
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '17'
           distribution: 'temurin'
@@ -77,9 +77,9 @@ jobs:
       matrix:
         module: ${{fromJSON(needs.initiate.outputs.matrix)}}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '17'
           distribution: 'temurin'
@@ -89,7 +89,7 @@ jobs:
         env:
           JVM_OPTS: -Xmx3200m
       - name: Publish test results
-        uses: EnricoMi/publish-unit-test-result-action@v1
+        uses: EnricoMi/publish-unit-test-result-action@v2
         if: always()
         with:
           files: "**/target/**/test-reports/*.xml"
@@ -107,9 +107,9 @@ jobs:
       matrix:
         module: ${{fromJSON(needs.initiate.outputs.it_matrix)}}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '17'
           distribution: 'temurin'
@@ -119,7 +119,7 @@ jobs:
         env:
           JVM_OPTS: -Xmx3200m
       - name: Publish test results
-        uses: EnricoMi/publish-unit-test-result-action@v1
+        uses: EnricoMi/publish-unit-test-result-action@v2
         if: always()
         with:
           files: "**/target/**/it-reports/*.xml"
@@ -129,20 +129,18 @@ jobs:
   build-and-cache-assembly:
     needs:
       - initiate
-      - test
-      - integration-test
     timeout-minutes: 30
     runs-on: ubuntu-latest
     strategy:
       matrix:
         module: ${{fromJSON(needs.initiate.outputs.matrix)}}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         name: Checkout repository
         with:
           fetch-depth: 0
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '17'
           distribution: 'temurin'
@@ -162,7 +160,7 @@ jobs:
           VERSION: ${{ steps.version.outputs.version }}
         run: sbt "project ${{ matrix.module }};set assembly / test := {}" assembly
       - name: Cache assembly
-        uses: actions/cache/save@v3
+        uses: actions/cache/save@v4
         with:
           path: ~/**/target/libs/*.jar
           key: assembly-${{ matrix.module }}-${{ github.run_id }}
@@ -178,15 +176,15 @@ jobs:
         module: ${{fromJSON(needs.initiate.outputs.fun_matrix)}}
         connectImageVersion: [7.3.1, 6.2.2]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '17'
           distribution: 'temurin'
           cache: 'sbt'
       - name: Restore assembly
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: ~/**/target/libs/*.jar
           key: assembly-${{ matrix.module }}-${{ github.run_id }}
@@ -197,7 +195,7 @@ jobs:
           JVM_OPTS: -Xmx3200m
           CONNECT_IMAGE_VERSION: ${{matrix.connectImageVersion}}
       - name: Publish test results
-        uses: EnricoMi/publish-unit-test-result-action@v1
+        uses: EnricoMi/publish-unit-test-result-action@v2
         if: always()
         with:
           files: "**/target/**/test-reports/*.xml"
@@ -215,17 +213,24 @@ jobs:
         module: ${{fromJSON(needs.initiate.outputs.dep_check_matrix)}}
     steps:
       - name: Restore assembly
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: ~/**/target/libs/*.jar
           key: assembly-${{ matrix.module }}-${{ github.run_id }}
           fail-on-cache-miss: true
+      - name: Extract branch name
+        shell: bash
+        run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
+        id: extract_branch
       - name: Dependency Check
-        uses: dependency-check/Dependency-Check_Action@1.1.0
+        uses: dependency-check/Dependency-Check_Action@main
         with:
           project: kafka-connect-${{matrix.module}}-deps
           path: kafka-connect-${{matrix.module}}/target/libs/
           format: 'HTML'
+          args: >-
+              --failOnCVSS 5
+              --suppression https://raw.githubusercontent.com/lensesio/stream-reactor/${{ steps.extract_branch.outputs.branch }}/suppression.xml
       - name: Upload Test results
         uses: actions/upload-artifact@master
         with:
@@ -243,21 +248,21 @@ jobs:
       matrix:
         module: ${{fromJSON(needs.initiate.outputs.matrix)}}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '17'
           distribution: 'temurin'
           cache: 'sbt'
       - name: Restore assembly
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: ~/**/target/libs/*.jar
           key: assembly-${{ matrix.module }}-${{ github.run_id }}
           fail-on-cache-miss: true
       - name: Upload artifact
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: assembly-${{ matrix.module }}-${{ github.run_id }}
           path: ~/**/target/libs/*.jar
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -14,7 +14,7 @@ jobs:
       tag: ${{ steps.get_tag.outputs.tag }}
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
@@ -47,7 +47,7 @@ jobs:
     timeout-minutes: 15
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Install and Update Antivirus Software
         run: |
           # Install ClamAV
@@ -60,7 +60,7 @@ jobs:
           clamscan --recursive --alert-broken --alert-encrypted \
             --alert-encrypted-archive --alert-exceeds-max --detect-pua .
       - name: Cache Antivirus Database
-        uses: actions/cache/save@v3
+        uses: actions/cache/save@v4
         with:
           path: /var/lib/clamav
           key: clamav-database-${{ github.run_id }}
@@ -75,9 +75,9 @@ jobs:
       matrix:
         module: ${{fromJSON(needs.initiate.outputs.dep_check_matrix)}}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Restore assembly
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: ~/**/target/libs/*.jar
           key: assembly-${{ matrix.module }}-${{ github.run_id }}
@@ -89,7 +89,7 @@ jobs:
           sudo systemctl stop clamav-freshclam
           sudo chmod 777 /var/lib/clamav
       - name: Restore Antivirus Database
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: /var/lib/clamav
           key: clamav-database-${{ github.run_id }}
@@ -121,16 +121,16 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       - name: Set up JDK 17
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '17'
           distribution: 'temurin'
           cache: 'sbt'
 
       - name: Uncache assembly
-        uses: actions/cache/restore@v3
+        uses: actions/cache/restore@v4
         with:
           path: |
             ~/**/target/libs/*.jar

diff --git a/.scalafmt.conf b/.scalafmt.conf
@@ -1,4 +1,4 @@
-version=2.6.1
+version=2.6.4
 maxColumn = 120
 preset = IntelliJ
 align.preset = most

diff --git a/build.sbt b/build.sbt
@@ -1,3 +1,4 @@
+import Dependencies.Versions
 import Dependencies.globalExcludeDeps
 import Dependencies.gson
 import Dependencies.bouncyCastle
@@ -149,7 +150,7 @@ lazy val `azure-datalake` = (project in file("kafka-connect-azure-datalake"))
         ),
       ),
   )
-  .configureAssembly(false)
+  .configureAssembly(true)
   .configureTests(baseTestDeps)
   //.configureIntegrationTests(kafkaConnectAzureDatalakeTestDeps)
   //.configureFunctionalTests(kafkaConnectAzureDatalakeFuncTestDeps)
@@ -293,7 +294,7 @@ lazy val opensearch = (project in file("kafka-connect-opensearch"))
 
 lazy val http = (project in file("kafka-connect-http"))
   .dependsOn(common)
-  //.dependsOn(`test-common` % "fun->compile")
+  .dependsOn(`test-common` % "fun->compile")
   .settings(
     settings ++
       Seq(
@@ -310,7 +311,7 @@ lazy val http = (project in file("kafka-connect-http"))
   .configureAssembly(false)
   .configureTests(baseTestDeps ++ kafkaConnectHttpTestDeps)
   .configureIntegrationTests(baseTestDeps ++ kafkaConnectHttpTestDeps)
-  //.configureFunctionalTests(kafkaConnectS3FuncTestDeps)
+  .configureFunctionalTests()
   .enablePlugins(PackPlugin, ProtocPlugin)
 
 lazy val influxdb = (project in file("kafka-connect-influxdb"))
@@ -350,14 +351,13 @@ lazy val jms = (project in file("kafka-connect-jms"))
         ),
         Compile / PB.protoSources := Seq(sourceDirectory.value / "test" / "resources" / "example"),
         Compile / PB.targets := Seq(
-          PB.gens.java -> (Test / sourceManaged).value,
+          PB.gens.java(Versions.googleProtobufVersion) -> (Test / sourceManaged).value,
         ),
       ),
   )
   .configureAssembly(true)
   .configureTests(kafkaConnectJmsTestDeps)
   .configureIntegrationTests(kafkaConnectJmsTestDeps)
-  //.configureFunctionalTests(kafkaConnectS3FuncTestDeps)
   .disableParallel()
   .enablePlugins(PackPlugin, ProtocPlugin)
 
@@ -474,12 +474,6 @@ addCommandAlias(
 addCommandAlias("fullTest", ";test;it:test;fun:test")
 addCommandAlias("fullCoverageTest", ";coverage;test;it:test;coverageReport;coverageAggregate")
 
-dependencyCheckFormats := Seq("XML", "HTML")
-dependencyCheckNodeAnalyzerEnabled := Some(false)
-dependencyCheckNodeAuditAnalyzerEnabled := Some(false)
-dependencyCheckNPMCPEAnalyzerEnabled := Some(false)
-dependencyCheckRetireJSAnalyzerEnabled := Some(false)
-
 excludeDependencies ++= globalExcludeDeps
 
 val generateModulesList         = taskKey[Seq[File]]("generateModulesList")

diff --git a/kafka-connect-aws-s3/src/fun/scala/io/lenses/streamreactor/connect/S3CompressionTest.scala b/kafka-connect-aws-s3/src/fun/scala/io/lenses/streamreactor/connect/S3CompressionTest.scala
@@ -11,7 +11,6 @@ import _root_.io.lenses.streamreactor.connect.testcontainers.scalatest.StreamRea
 import cats.effect.IO
 import cats.effect.testing.scalatest.AsyncIOSpec
 import cats.implicits._
-import com.datastax.driver.core.utils.UUIDs
 import com.typesafe.scalalogging.LazyLogging
 import org.apache.avro.generic.GenericRecord
 import org.apache.kafka.clients.producer.ProducerRecord
@@ -23,6 +22,7 @@ import org.scalatest.prop.TableDrivenPropertyChecks
 import org.scalatest.prop.TableFor3
 import software.amazon.awssdk.services.s3.model._
 
+import java.util.UUID
 import scala.jdk.CollectionConverters.ListHasAsScala
 import scala.util.Random
 
@@ -98,8 +98,8 @@ class S3CompressionTest
           case (s3Client, producer) =>
             IO {
               // Write records to
-              val order  = Order(1, "OP-DAX-P-20150201-95.7", 94.2, 100, UUIDs.timeBased.toString)
-              val record = order.toRecord(order)
+              val order  = Order(1, "OP-DAX-P-20150201-95.7", 94.2, 100, UUID.randomUUID().toString)
+              val record = order.toRecord
 
               producer.send(new ProducerRecord[String, GenericRecord](topic, record)).get
               producer.flush()

diff --git a/kafka-connect-aws-s3/src/fun/scala/io/lenses/streamreactor/connect/S3Test.scala b/kafka-connect-aws-s3/src/fun/scala/io/lenses/streamreactor/connect/S3Test.scala
@@ -11,7 +11,6 @@ import _root_.io.lenses.streamreactor.connect.testcontainers.S3Container
 import _root_.io.lenses.streamreactor.connect.testcontainers.SchemaRegistryContainer
 import cats.effect.IO
 import cats.effect.testing.scalatest.AsyncIOSpec
-import com.datastax.driver.core.utils.UUIDs
 import com.typesafe.scalalogging.LazyLogging
 import org.apache.kafka.clients.producer.ProducerRecord
 import org.apache.kafka.common.serialization.StringSerializer
@@ -21,6 +20,8 @@ import org.scalatest.matchers.should.Matchers
 import org.scalatest.prop.TableDrivenPropertyChecks
 import software.amazon.awssdk.services.s3.model._
 
+import java.util.UUID
+
 class S3Test
     extends AsyncFlatSpec
     with AsyncIOSpec
@@ -52,7 +53,7 @@ class S3Test
 
   it should "sink records" in {
 
-    val order = Order(1, "OP-DAX-P-20150201-95.7", 94.2, 100, UUIDs.timeBased.toString)
+    val order = Order(1, "OP-DAX-P-20150201-95.7", 94.2, 100, UUID.randomUUID().toString)
 
     val resources = for {
       s3Client <- createS3ClientResource(container.identity, container.getEndpointUrl)

diff --git a/...s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3AvroWriterManagerTest.scala b/...s3/src/it/scala/io/lenses/streamreactor/connect/aws/s3/sink/S3AvroWriterManagerTest.scala
@@ -35,6 +35,7 @@ import io.lenses.streamreactor.connect.cloud.common.model.Offset
 import io.lenses.streamreactor.connect.cloud.common.model.Topic
 import io.lenses.streamreactor.connect.cloud.common.model.TopicPartitionOffset
 import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocation
+import io.lenses.streamreactor.connect.cloud.common.model.location.CloudLocationValidator
 import io.lenses.streamreactor.connect.cloud.common.sink.WriterManagerCreator
 import io.lenses.streamreactor.connect.cloud.common.sink.commit.CommitPolicy
 import io.lenses.streamreactor.connect.cloud.common.sink.commit.Count
@@ -70,8 +71,8 @@ class S3AvroWriterManagerTest extends AnyFlatSpec with Matchers with S3ProxyCont
   private val PathPrefix       = "streamReactorBackups"
   private val avroFormatReader = new AvroFormatReader
 
-  private implicit val cloudLocationValidator = S3LocationValidator
-  private val bucketAndPrefix                 = CloudLocation(BucketName, PathPrefix.some)
+  private implicit val cloudLocationValidator: CloudLocationValidator = S3LocationValidator
+  private val bucketAndPrefix = CloudLocation(BucketName, PathPrefix.some)
   private def avroConfig = S3SinkConfig(
     S3ConnectionConfig(
       None,

diff --git a/...scala/io/lenses/streamreactor/connect/aws/s3/sink/config/S3SinkConfigDefBuilderTest.scala b/...scala/io/lenses/streamreactor/connect/aws/s3/sink/config/S3SinkConfigDefBuilderTest.scala
@@ -40,7 +40,7 @@ class S3SinkConfigDefBuilderTest extends AnyFlatSpec with MockitoSugar with Matc
   val BucketName = "mybucket"
 
   private implicit val cloudLocationValidator: CloudLocationValidator = S3LocationValidator
-  private implicit val connectorTaskId = ConnectorTaskId("connector", 1, 0)
+  private implicit val connectorTaskId:        ConnectorTaskId        = ConnectorTaskId("connector", 1, 0)
 
   "S3SinkConfigDefBuilder" should "respect defined properties" in {
     val props = Map(