diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 2f3d186..15246c4 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -28,7 +28,7 @@ jobs: - name: Publish collector locally run: | - git clone --branch 3.1.0_cross_scala_2.12 --depth 1 https://github.com/snowplow/stream-collector.git + git clone --branch 3.2.0 --depth 1 https://github.com/snowplow/stream-collector.git cd stream-collector sbt +publishLocal diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index faa6a6d..39aff62 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -18,7 +18,7 @@ jobs: - name: Publish collector locally run: | - git clone --branch 3.1.0_cross_scala_2.12 --depth 1 https://github.com/snowplow/stream-collector.git + git clone --branch 3.2.0 --depth 1 https://github.com/snowplow/stream-collector.git cd stream-collector sbt +publishLocal diff --git a/README.md b/README.md index f1e0a51..0ef4d5d 100644 --- a/README.md +++ b/README.md @@ -23,9 +23,9 @@ Assuming [Git][git] and [sbt][sbt]: git clone git@github.com:snowplow-incubator/snowplow-micro.git cd snowplow-micro -git clone --branch 2.8.1 --depth 1 git@github.com:snowplow/stream-collector.git +git clone --branch 3.2.0 --depth 1 git@github.com:snowplow/stream-collector.git cd stream-collector -sbt publishLocal && cd .. +sbt +publishLocal && cd .. sbt test ``` diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 70eb3c3..1c02f18 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -16,7 +16,7 @@ object Dependencies { object V { // Snowplow - val snowplowStreamCollector = "3.1.3-rc1" + val snowplowStreamCollector = "3.2.0" val snowplowCommonEnrich = "4.1.0" val http4sCirce = "0.23.23" diff --git a/project/plugins.sbt b/project/plugins.sbt index c06de5f..f375a39 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,8 +1,6 @@ -logLevel := Level.Warn - -addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.0.0") -addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.8.2") -addSbtPlugin("com.github.sbt" % "sbt-native-packager" % "1.9.3") -addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.10.0") -addSbtPlugin("com.dwijnand" % "sbt-dynver" % "4.1.1") -addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.10.0") +addSbtPlugin("io.github.davidgregory084" % "sbt-tpolecat" % "0.4.2") +addSbtPlugin("com.snowplowanalytics" % "sbt-snowplow-release" % "0.3.1") +addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.11.0") +addSbtPlugin("com.dwijnand" % "sbt-dynver" % "4.1.1") +addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "1.0.0") +addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.10.0") diff --git a/src/main/scala/com.snowplowanalytics.snowplow.micro/MicroHttpServer.scala b/src/main/scala/com.snowplowanalytics.snowplow.micro/MicroHttpServer.scala index 2664719..39a1687 100644 --- a/src/main/scala/com.snowplowanalytics.snowplow.micro/MicroHttpServer.scala +++ b/src/main/scala/com.snowplowanalytics.snowplow.micro/MicroHttpServer.scala @@ -15,12 +15,13 @@ import com.avast.datadog4s.api.Tag import com.avast.datadog4s.extension.http4s.DatadogMetricsOps import com.avast.datadog4s.{StatsDMetricFactory, StatsDMetricFactoryConfig} import com.snowplowanalytics.snowplow.collector.core.{Config => CollectorConfig} -import com.snowplowanalytics.snowplow.micro.Configuration.MicroConfig +import com.snowplowanalytics.snowplow.micro.Configuration.{MicroConfig, SinkConfig} import org.http4s.blaze.server.BlazeServerBuilder import org.http4s.headers.`Strict-Transport-Security` import org.http4s.server.Server -import org.http4s.server.middleware.{HSTS, Metrics} +import org.http4s.server.middleware.{HSTS, Metrics, Timeout, Logger => LoggerMiddleware} import org.http4s.{HttpApp, HttpRoutes} +import org.typelevel.ci.CIString import org.typelevel.log4cats.Logger import org.typelevel.log4cats.slf4j.Slf4jLogger @@ -44,23 +45,44 @@ object MicroHttpServer { } yield () private def buildHTTPServer(routes: HttpRoutes[IO], config: MicroConfig): Resource[IO, Server] = - builder(routes, config) + builder(routes, config.collector) .bindSocketAddress(new InetSocketAddress(config.collector.port)) .resource private def buildHTTPSServer(routes: HttpRoutes[IO], config: MicroConfig, sslContext: SSLContext): Resource[IO, Server] = - builder(routes, config) + builder(routes, config.collector) .bindSocketAddress(new InetSocketAddress(config.collector.ssl.port)) .withSslContext(sslContext) .resource - private def builder(routes: HttpRoutes[IO], config: MicroConfig): BlazeServerBuilder[IO] = { + private def builder(routes: HttpRoutes[IO], config: CollectorConfig[SinkConfig]): BlazeServerBuilder[IO] = { BlazeServerBuilder[IO] - .withHttpApp(hstsMiddleware(config.collector.hsts, routes.orNotFound)) - .withIdleTimeout(config.collector.networking.idleTimeout) - .withMaxConnections(config.collector.networking.maxConnections) + .withHttpApp( + loggerMiddleware(timeoutMiddleware(hstsMiddleware(config.hsts, routes.orNotFound), config.networking), config.debug.http) + ) + .withIdleTimeout(config.networking.idleTimeout) + .withMaxConnections(config.networking.maxConnections) + .withResponseHeaderTimeout(config.networking.responseHeaderTimeout) + .withLengthLimits( + maxRequestLineLen = config.networking.maxRequestLineLength, + maxHeadersLen = config.networking.maxHeadersLength + ) } + private def loggerMiddleware(routes: HttpApp[IO], config: CollectorConfig.Debug.Http): HttpApp[IO] = + if (config.enable) { + LoggerMiddleware.httpApp[IO]( + logHeaders = config.logHeaders, + logBody = config.logBody, + redactHeadersWhen = config.redactHeaders.map(CIString(_)).contains(_), + logAction = Some((msg: String) => Logger[IO].debug(msg)) + )(routes) + } else routes + + private def timeoutMiddleware(routes: HttpApp[IO], networking: CollectorConfig.Networking): HttpApp[IO] = + Timeout.httpApp[IO](timeout = networking.responseHeaderTimeout)(routes) + + private def createMetricsMiddleware(routes: HttpRoutes[IO], metricsCollectorConfig: CollectorConfig.Metrics): Resource[IO, HttpRoutes[IO]] = if (metricsCollectorConfig.statsd.enabled) { diff --git a/src/main/scala/com.snowplowanalytics.snowplow.micro/Run.scala b/src/main/scala/com.snowplowanalytics.snowplow.micro/Run.scala index 81e6a9e..96949a4 100644 --- a/src/main/scala/com.snowplowanalytics.snowplow.micro/Run.scala +++ b/src/main/scala/com.snowplowanalytics.snowplow.micro/Run.scala @@ -68,6 +68,7 @@ object Run { config.collector.enableDefaultRedirect, config.collector.rootResponse.enabled, config.collector.crossDomain.enabled, + config.collector.networking.bodyReadTimeout, collectorService ).value