RDB Loader: deprecate steps (close #625)

snowplow · Dec 1, 2021 · bda93a4 · bda93a4
1 parent 324c671
commit bda93a4
Show file tree

Hide file tree

Showing 4 changed files with 8 additions and 10 deletions.
diff --git a/config/loader.config.reference.hocon b/config/loader.config.reference.hocon
@@ -33,10 +33,6 @@
     "maxError": 10
   },
 
-  # Additional steps. analyze, vacuum and transit_load are valid values
-  # Optional, default value []
-  "steps": ["analyze"],
-
   # Observability and reporting options
   "monitoring": {
     # Snowplow tracking (optional)

diff --git a/modules/loader/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/Main.scala b/modules/loader/src/main/scala/com/snowplowanalytics/snowplow/rdbloader/Main.scala
@@ -64,9 +64,11 @@ object Main extends IOApp {
     val folderMonitoring: Stream[F, Unit] =
       FolderMonitoring.run[F](cli.config.monitoring.folders, cli.config.storage, control.isBusy)
 
+    // TODO: Currently, steps are deactivated with making them empty.
+    // Remove them from the codebase properly.
     Stream.eval_(Manifest.initialize[F](cli.config.storage)) ++
       DataDiscovery
-        .discover[F](cli.config, control.state)
+        .discover[F](cli.config.copy(steps = Set.empty), control.state)
         .pauseWhen[F](control.isBusy)
         .evalMap { discovery =>
           val prepare = for {
@@ -75,7 +77,7 @@ object Main extends IOApp {
           } yield ()
 
           val loading: F[Unit] = prepare.use { _ =>
-            load[F](cli.config, control.state.setStage, discovery).rethrowT *> control.state.update(_.incrementLoaded)
+            load[F](cli.config.copy(steps = Set.empty), control.state.setStage, discovery).rethrowT *> control.state.update(_.incrementLoaded)
           }
 
           // Catches both connection acquisition and loading errors

diff --git a/...es/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/config/ConfigSpec.scala b/...es/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/config/ConfigSpec.scala
@@ -94,7 +94,7 @@ object ConfigSpec {
     10,
     None
   )
-  val exampleSteps: Set[Step] = Set(Step.Analyze)
+  val exampleSteps: Set[Step] = Set()
 
   def getConfig[A](confPath: String, parse: String => EitherT[IO, String, A]): Either[String, A] =
     parse(readResource(confPath)).value.unsafeRunSync()

diff --git a/...les/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/loading/LoadSpec.scala b/...les/loader/src/test/scala/com/snowplowanalytics/snowplow/rdbloader/loading/LoadSpec.scala
@@ -65,7 +65,7 @@ class LoadSpec extends Specification {
         LogEntry.Sql(Statement.Commit)
       )
 
-      val result = Load.load[Pure](SpecHelpers.validCliConfig.config, LoadSpec.setStageNoOp, message).value.runS
+      val result = Load.load[Pure](SpecHelpers.validCliConfig.config.copy(steps = Set(Step.Analyze)), LoadSpec.setStageNoOp, message).value.runS
 
       result.getLog must beEqualTo(expected)
     }
@@ -96,7 +96,7 @@ class LoadSpec extends Specification {
         LogEntry.Sql(Statement.Commit)
       )
 
-      val result = Load.load[Pure](SpecHelpers.validCliConfig.config, LoadSpec.setStageNoOp, message).value.runS
+      val result = Load.load[Pure](SpecHelpers.validCliConfig.config.copy(steps = Set(Step.Analyze)), LoadSpec.setStageNoOp, message).value.runS
 
       result.getLog must beEqualTo(expected)
     }
@@ -158,7 +158,7 @@ class LoadSpec extends Specification {
         LogEntry.Sql(Statement.Analyze("atomic.com_acme_json_context_1")),
         LogEntry.Sql(Statement.Commit)
       )
-      val result = Load.load[Pure](SpecHelpers.validCliConfig.config, LoadSpec.setStageNoOp, message).value.runS
+      val result = Load.load[Pure](SpecHelpers.validCliConfig.config.copy(steps = Set(Step.Analyze)), LoadSpec.setStageNoOp, message).value.runS
 
       result.getLog must beEqualTo(expected)
     }