Collector 3.x and Enrich 4.x

snowplow-incubator · Feb 15, 2024 · 20c954a · 20c954a
1 parent 2b4150d
commit 20c954a
Show file tree

Hide file tree

Showing 20 changed files with 860 additions and 1,009 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -28,9 +28,9 @@ jobs:
 
     - name: Publish collector locally
       run: |
-        git clone --branch 2.8.1 --depth 1 https://github.com/snowplow/stream-collector.git
+        git clone --branch 3.1.0_cross_scala_2.12 --depth 1 https://github.com/snowplow/stream-collector.git
         cd stream-collector
-        sbt publishLocal
+        sbt +publishLocal
 
     - name: Run sbt
       run: sbt test

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -18,9 +18,9 @@ jobs:
 
     - name: Publish collector locally
       run: |
-        git clone --branch 2.8.1 --depth 1 https://github.com/snowplow/stream-collector.git
+        git clone --branch 3.1.0_cross_scala_2.12 --depth 1 https://github.com/snowplow/stream-collector.git
         cd stream-collector
-        sbt publishLocal
+        sbt +publishLocal
 
     - name: Run sbt
       run: sbt test
diff --git a/build.sbt b/build.sbt
@@ -6,7 +6,9 @@
   * Copyright (c) 2019-2022 Snowplow Analytics Ltd. All rights reserved.
   */
 import com.typesafe.sbt.packager.MappingsHelper.directory
-import com.typesafe.sbt.packager.docker._
+import com.typesafe.sbt.packager.docker.*
+
+import scala.collection.Seq
 
 lazy val buildSettings = Seq(    
   name := "snowplow-micro",
@@ -27,30 +29,30 @@ lazy val dependencies = Seq(
   libraryDependencies ++= Seq(
     Dependencies.snowplowStreamCollector,
     Dependencies.snowplowCommonEnrich,
+    Dependencies.decline,
+    Dependencies.http4sCirce,
     Dependencies.circeJawn,
     Dependencies.circeGeneric,
     Dependencies.specs2,
+    Dependencies.specs2CE,
     Dependencies.badRows
   )
 )
 
-lazy val exclusions = Seq(
-  excludeDependencies ++= Dependencies.exclusions
-)
-
 lazy val buildInfoSettings = Seq(
-  buildInfoKeys := Seq[BuildInfoKey](organization, name, version, scalaVersion),
-  buildInfoPackage := "buildinfo"
+  buildInfoKeys := Seq[BuildInfoKey](name, moduleName, dockerAlias, version, "shortName" -> "micro-ssc"),
+  buildInfoPackage := "com.snowplowanalytics.snowplow.micro",
+  buildInfoOptions += BuildInfoOption.Traits("com.snowplowanalytics.snowplow.collector.core.AppInfo")
 )
 
+
 lazy val dynVerSettings = Seq(
   ThisBuild / dynverVTagPrefix := false, // Otherwise git tags required to have v-prefix
   ThisBuild / dynverSeparator := "-"     // to be compatible with docker
 )
 
 lazy val commonSettings = 
   dependencies ++
-  exclusions ++
   buildSettings ++
   buildInfoSettings ++
   dynVerSettings ++

diff --git a/project/Dependencies.scala b/project/Dependencies.scala
@@ -16,33 +16,36 @@ object Dependencies {
 
   object V {
     // Snowplow
-    val snowplowStreamCollector = "2.8.1"
-    val snowplowCommonEnrich    = "3.8.0"
+    val snowplowStreamCollector = "3.1.1-rc1"
+    val snowplowCommonEnrich    = "4.0.1"
+    val http4sCirce             = "0.23.23"
 
+    val decline          = "2.4.1"
+
     // circe
     val circe = "0.14.2"
 
     // specs2
     val specs2        = "4.12.2"
+    val specs2CE      = "1.5.0"
 
     // force versions of transitive dependencies
     val badRows   = "2.2.0"
   }
 
-  val exclusions = Seq(
-    "org.apache.tomcat.embed" % "tomcat-embed-core"
-  )
-
-  // Snowplow stream collector
-  val snowplowStreamCollector = "com.snowplowanalytics" %% "snowplow-stream-collector-core" % V.snowplowStreamCollector
-  val snowplowCommonEnrich    = "com.snowplowanalytics" %% "snowplow-common-enrich"         % V.snowplowCommonEnrich
+  val snowplowStreamCollector = "com.snowplowanalytics" %% "snowplow-stream-collector-http4s-core" % V.snowplowStreamCollector 
+  val snowplowCommonEnrich    = "com.snowplowanalytics" %% "snowplow-common-enrich"                % V.snowplowCommonEnrich 
+
+  val http4sCirce = "org.http4s"    %% "http4s-circe"   % V.http4sCirce 
+  val decline     = "com.monovore"  %% "decline-effect" % V.decline
 
   // circe
   val circeJawn    = "io.circe" %% "circe-jawn"    % V.circe
   val circeGeneric = "io.circe" %% "circe-generic" % V.circe
 
   // specs2
-  val specs2       = "org.specs2"    %% "specs2-core"   % V.specs2       % Test
+  val specs2       = "org.specs2"     %% "specs2-core"                % V.specs2    % Test
+  val specs2CE     = "org.typelevel"  %% "cats-effect-testing-specs2" % V.specs2CE  % Test
 
   // transitive
   val badRows          = "com.snowplowanalytics"           %% "snowplow-badrows"        % V.badRows

diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf
diff --git a/src/main/resources/collector-micro.conf b/src/main/resources/collector-micro.conf
@@ -0,0 +1,18 @@
+collector {
+  license {
+    accept = true
+  }
+  interface = "0.0.0.0"
+  port = 8080
+
+  streams {
+    good = "good"
+    bad  = "bad" 
+    buffer {
+      byteLimit = 3145728
+      recordLimit = 500
+      timeLimit = 5000
+    }
+    sink {}
+  }
+}
diff --git a/...main/resources/default-iglu-resolver.json → ...main/resources/default-iglu-resolver.conf b/...main/resources/default-iglu-resolver.json → ...main/resources/default-iglu-resolver.conf
diff --git a/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/ValueConverter.scala b/src/main/scala/com.snowplowanalytics.snowplow.analytics.scalasdk/decode/ValueConverter.scala
@@ -39,7 +39,7 @@ object ValueConverter {
     ofFunc(_ => f(_).asRight)
 
   implicit def valueDecoderCase[A](implicit decoder: ValueDecoder[A]): Aux[Option[String], A] =
-    ofFunc(key => x => decoder.parse(key, x.getOrElse("")))
+    ofFunc(key => x => decoder.parse(key, x.getOrElse(""), None))
 
   implicit def floatDoubleCase: Aux[Option[Float], Option[Double]] =
     simple(_.map(_.toDouble))