Refactor SdkBindindData factory methods (#193)

* WIP Signed-off-by: Nelson Arapé <[email protected]> * Refactor SdkBindingData ans SdkType ... based on SdkLiteralType Still missing: * conversions * javadocs * check error messages Signed-off-by: Nelson Arapé <[email protected]> * WIP for converting lists Signed-off-by: Nelson Arapé <[email protected]> * WIP transform lists Signed-off-by: Lucía Pasarin <[email protected]> * First test compiles Signed-off-by: Nelson Arapé <[email protected]> * Now ready for review Signed-off-by: Nelson Arapé <[email protected]> * Format Signed-off-by: Nelson Arapé <[email protected]> * Fix invalid javadoc Signed-off-by: Nelson Arapé <[email protected]> * Address first round of feedback Signed-off-by: Nelson Arapé <[email protected]> * Rename SdkBindingDatas to SdkBindingDataFactory and change ofX methods Signed-off-by: Pablo Casares Crespo <[email protected]> * Review changes Signed-off-by: Andres Gomez Ferrer <[email protected]> * Fix verify Signed-off-by: Andres Gomez Ferrer <[email protected]> * Fix integration test Signed-off-by: Andres Gomez Ferrer <[email protected]> --------- Signed-off-by: Nelson Arapé <[email protected]> Signed-off-by: Lucía Pasarin <[email protected]> Signed-off-by: Pablo Casares Crespo <[email protected]> Signed-off-by: Andres Gomez Ferrer <[email protected]> Co-authored-by: Lucía Pasarin <[email protected]> Co-authored-by: Pablo Casares Crespo <[email protected]> Co-authored-by: Andres Gomez Ferrer <[email protected]>
flyteorg · Feb 14, 2023 · 305495e · 305495e
1 parent 031c6c5
commit 305495e
Show file tree

Hide file tree

Showing 85 changed files with 3,142 additions and 2,152 deletions.
diff --git a/...ekit-examples-scala/src/main/scala/org/flyte/examples/flytekitscala/AddQuestionTask.scala b/...ekit-examples-scala/src/main/scala/org/flyte/examples/flytekitscala/AddQuestionTask.scala
@@ -16,9 +16,9 @@
  */
 package org.flyte.examples.flytekitscala
 
-import org.flyte.flytekit.{SdkBindingData, SdkRunnableTask, SdkTransform}
+import org.flyte.flytekit.{SdkBindingData, SdkRunnableTask}
 import org.flyte.flytekitscala.SdkScalaType
-import org.flyte.flytekitscala.SdkBindingData.ofString
+import org.flyte.flytekitscala.SdkBindingDataFactory
 
 case class AddQuestionTaskInput(greeting: SdkBindingData[String])
 case class AddQuestionTaskOutput(greeting: SdkBindingData[String])
@@ -44,5 +44,7 @@ class AddQuestionTask
     *   the updated greeting message
     */
   override def run(input: AddQuestionTaskInput): AddQuestionTaskOutput =
-    AddQuestionTaskOutput(ofString(s"${input.greeting.get} How are you?"))
+    AddQuestionTaskOutput(
+      SdkBindingDataFactory.of(s"${input.greeting.get} How are you?")
+    )
 }
diff --git a/...ples-scala/src/main/scala/org/flyte/examples/flytekitscala/DynamicFibonacciWorkflow.scala b/...ples-scala/src/main/scala/org/flyte/examples/flytekitscala/DynamicFibonacciWorkflow.scala
@@ -16,7 +16,7 @@
  */
 package org.flyte.examples.flytekitscala
 
-import org.flyte.flytekit.{SdkBindingData, SdkWorkflow, SdkWorkflowBuilder}
+import org.flyte.flytekit.SdkBindingData
 import org.flyte.flytekitscala.{
   SdkScalaType,
   SdkScalaWorkflow,

diff --git a/...-scala/src/main/scala/org/flyte/examples/flytekitscala/DynamicFibonacciWorkflowTask.scala b/...-scala/src/main/scala/org/flyte/examples/flytekitscala/DynamicFibonacciWorkflowTask.scala
@@ -24,7 +24,7 @@ import org.flyte.flytekit.{
 import org.flyte.flytekitscala.SdkScalaType
 
 import scala.annotation.tailrec
-import org.flyte.flytekitscala.SdkBindingData._
+import org.flyte.flytekitscala.SdkBindingDataFactory._
 
 case class DynamicFibonacciWorkflowTaskInput(n: SdkBindingData[Long])
 case class DynamicFibonacciWorkflowTaskOutput(output: SdkBindingData[Long])
@@ -64,9 +64,9 @@ class DynamicFibonacciWorkflowTask
 
     require(input.n.get > 0, "n < 0")
     val value = if (input.n.get == 0) {
-      ofInteger(0)
+      of(0)
     } else {
-      fib(1, ofInteger(1), ofInteger(0))
+      fib(1, of(1), of(0))
     }
     DynamicFibonacciWorkflowTaskOutput(value)
   }

diff --git a/flytekit-examples-scala/src/main/scala/org/flyte/examples/flytekitscala/GreetTask.scala b/flytekit-examples-scala/src/main/scala/org/flyte/examples/flytekitscala/GreetTask.scala
@@ -17,8 +17,11 @@
 package org.flyte.examples.flytekitscala
 
 import org.flyte.flytekit.{SdkBindingData, SdkRunnableTask, SdkTransform}
-import org.flyte.flytekitscala.{Description, SdkScalaType}
-import org.flyte.flytekitscala.SdkBindingData._
+import org.flyte.flytekitscala.{
+  Description,
+  SdkBindingDataFactory,
+  SdkScalaType
+}
 
 case class GreetTaskInput(
     @Description("the name of the person to be greeted")
@@ -47,5 +50,5 @@ class GreetTask
     *   the welcome message
     */
   override def run(input: GreetTaskInput): GreetTaskOutput =
-    GreetTaskOutput(ofString(s"Welcome, ${input.name.get()}!"))
+    GreetTaskOutput(SdkBindingDataFactory.of(s"Welcome, ${input.name.get()}!"))
 }
diff --git a/flytekit-examples-scala/src/main/scala/org/flyte/examples/flytekitscala/SumTask.scala b/flytekit-examples-scala/src/main/scala/org/flyte/examples/flytekitscala/SumTask.scala
@@ -18,7 +18,7 @@ package org.flyte.examples.flytekitscala
 
 import org.flyte.flytekit.{SdkBindingData, SdkRunnableTask, SdkTransform}
 import org.flyte.flytekitscala.{Description, SdkScalaType}
-import org.flyte.flytekitscala.SdkBindingData._
+import org.flyte.flytekitscala.SdkBindingDataFactory._
 
 case class SumTaskInput(
     @Description("First operand")
@@ -39,7 +39,7 @@ class SumTask
 
   override def run(input: SumTaskInput): SumTaskOutput = {
     val result = input.a.get + input.b.get
-    SumTaskOutput(ofInteger(result))
+    SumTaskOutput(of(result))
   }
 
   override def isCached: Boolean = true

diff --git a/...ekit-examples-scala/src/main/scala/org/flyte/examples/flytekitscala/WelcomeWorkflow.scala b/...ekit-examples-scala/src/main/scala/org/flyte/examples/flytekitscala/WelcomeWorkflow.scala
@@ -16,7 +16,7 @@
  */
 package org.flyte.examples.flytekitscala
 
-import org.flyte.flytekit.{SdkBindingData, SdkWorkflow, SdkWorkflowBuilder}
+import org.flyte.flytekit.SdkBindingData
 import org.flyte.flytekitscala.{
   SdkScalaType,
   SdkScalaWorkflow,

diff --git a/...-scala/src/main/scala/org/flyte/examples/flytekitscala/WorkflowWithRemoteLaunchPlan.scala b/...-scala/src/main/scala/org/flyte/examples/flytekitscala/WorkflowWithRemoteLaunchPlan.scala
@@ -17,7 +17,7 @@
 package org.flyte.examples.flytekitscala
 
 import org.flyte.flytekitscala.{
-  SdkBindingData,
+  SdkBindingDataFactory,
   SdkScalaType,
   SdkScalaWorkflow,
   SdkScalaWorkflowBuilder
@@ -33,8 +33,8 @@ class WorkflowWithRemoteLaunchPlan
       builder: SdkScalaWorkflowBuilder,
       input: RemoteLaunchPlanInput
   ): RemoteLaunchPlanOutput = {
-    val fib0 = SdkBindingData.ofInteger(0L)
-    val fib1 = SdkBindingData.ofInteger(1L)
+    val fib0 = SdkBindingDataFactory.of(0L)
+    val fib1 = SdkBindingDataFactory.of(1L)
 
     val fib5 = builder
       .apply(

diff --git a/...amples-scala/src/main/scala/org/flyte/examples/flytekitscala/WorkflowWithRemoteTask.scala b/...amples-scala/src/main/scala/org/flyte/examples/flytekitscala/WorkflowWithRemoteTask.scala
@@ -17,7 +17,7 @@
 package org.flyte.examples.flytekitscala
 
 import org.flyte.flytekitscala.{
-  SdkBindingData,
+  SdkBindingDataFactory,
   SdkScalaType,
   SdkScalaWorkflow,
   SdkScalaWorkflowBuilder
@@ -33,8 +33,8 @@ class WorkflowWithRemoteTask
       builder: SdkScalaWorkflowBuilder,
       input: RemoteSumTaskInput
   ): RemoteSumTaskOutput = {
-    val a = SdkBindingData.ofInteger(10)
-    val b = SdkBindingData.ofInteger(12)
+    val a = SdkBindingDataFactory.of(10)
+    val b = SdkBindingDataFactory.of(12)
 
     val c = builder
       .apply(new RemoteSumTask().create, RemoteSumTaskInput(a, b))

diff --git a/flytekit-examples/src/main/java/org/flyte/examples/AddQuestionTask.java b/flytekit-examples/src/main/java/org/flyte/examples/AddQuestionTask.java
@@ -19,6 +19,7 @@
 import com.google.auto.service.AutoService;
 import com.google.auto.value.AutoValue;
 import org.flyte.flytekit.SdkBindingData;
+import org.flyte.flytekit.SdkBindingDataFactory;
 import org.flyte.flytekit.SdkRunnableTask;
 import org.flyte.flytekit.jackson.JacksonSdkType;
 
@@ -75,6 +76,6 @@ public static Output create(SdkBindingData<String> greeting) {
   @Override
   public Output run(Input input) {
     return Output.create(
-        SdkBindingData.ofString(String.format("%s How are you?", input.greeting().get())));
+        SdkBindingDataFactory.of(String.format("%s How are you?", input.greeting().get())));
   }
 }
diff --git a/flytekit-examples/src/main/java/org/flyte/examples/AllInputsWorkflow.java b/flytekit-examples/src/main/java/org/flyte/examples/AllInputsWorkflow.java
@@ -26,6 +26,7 @@
 import java.util.Map;
 import org.flyte.examples.AllInputsTask.AutoAllInputsOutput;
 import org.flyte.flytekit.SdkBindingData;
+import org.flyte.flytekit.SdkBindingDataFactory;
 import org.flyte.flytekit.SdkNode;
 import org.flyte.flytekit.SdkTypes;
 import org.flyte.flytekit.SdkWorkflow;
@@ -50,16 +51,16 @@ public AllInputsWorkflowOutput expand(SdkWorkflowBuilder builder, Void noInput)
             "all-inputs",
             new AllInputsTask(),
             AllInputsTask.AutoAllInputsInput.create(
-                SdkBindingData.ofInteger(1L),
-                SdkBindingData.ofFloat(2),
-                SdkBindingData.ofString("test"),
-                SdkBindingData.ofBoolean(true),
-                SdkBindingData.ofDatetime(someInstant),
-                SdkBindingData.ofDuration(Duration.ofDays(1L)),
-                SdkBindingData.ofStringCollection(Arrays.asList("foo", "bar")),
-                SdkBindingData.ofStringMap(Map.of("test", "test")),
-                SdkBindingData.ofStringCollection(Collections.emptyList()),
-                SdkBindingData.ofIntegerMap(Collections.emptyMap())));
+                SdkBindingDataFactory.of(1L),
+                SdkBindingDataFactory.of(2.00),
+                SdkBindingDataFactory.of("test"),
+                SdkBindingDataFactory.of(true),
+                SdkBindingDataFactory.of(someInstant),
+                SdkBindingDataFactory.of(Duration.ofDays(1L)),
+                SdkBindingDataFactory.ofStringCollection(Arrays.asList("foo", "bar")),
+                SdkBindingDataFactory.ofStringMap(Map.of("test", "test")),
+                SdkBindingDataFactory.ofStringCollection(Collections.emptyList()),
+                SdkBindingDataFactory.ofIntegerMap(Collections.emptyMap())));
 
     AllInputsTask.AutoAllInputsOutput outputs = apply.getOutputs();
 

diff --git a/flytekit-examples/src/main/java/org/flyte/examples/BatchLookUpTask.java b/flytekit-examples/src/main/java/org/flyte/examples/BatchLookUpTask.java
@@ -22,6 +22,7 @@
 import java.util.Map;
 import java.util.stream.Collectors;
 import org.flyte.flytekit.SdkBindingData;
+import org.flyte.flytekit.SdkBindingDataFactory;
 import org.flyte.flytekit.SdkRunnableTask;
 import org.flyte.flytekit.jackson.JacksonSdkType;
 
@@ -46,7 +47,7 @@ public Output run(Input input) {
             .map(key -> input.keyValues().get().get(key))
             .collect(Collectors.toList());
 
-    return Output.create(SdkBindingData.ofStringCollection(foundValues));
+    return Output.create(SdkBindingDataFactory.ofStringCollection(foundValues));
   }
 
   @AutoValue

diff --git a/flytekit-examples/src/main/java/org/flyte/examples/ConditionalGreetingWorkflow.java b/flytekit-examples/src/main/java/org/flyte/examples/ConditionalGreetingWorkflow.java
@@ -16,11 +16,11 @@
  */
 package org.flyte.examples;
 
-import static org.flyte.flytekit.SdkBindingData.ofString;
 import static org.flyte.flytekit.SdkConditions.eq;
 
 import com.google.auto.service.AutoService;
 import org.flyte.flytekit.SdkBindingData;
+import org.flyte.flytekit.SdkBindingDataFactory;
 import org.flyte.flytekit.SdkConditions;
 import org.flyte.flytekit.SdkWorkflow;
 import org.flyte.flytekit.SdkWorkflowBuilder;
@@ -40,9 +40,9 @@ public GreetTask.Output expand(SdkWorkflowBuilder builder, GreetTask.Input input
                 "decide",
                 SdkConditions.when(
                         "when-empty",
-                        eq(input.name(), ofString("")),
+                        eq(input.name(), SdkBindingDataFactory.of("")),
                         new GreetTask(),
-                        GreetTask.Input.create(ofString("World")))
+                        GreetTask.Input.create(SdkBindingDataFactory.of("World")))
                     .otherwise(
                         "when-not-empty", new GreetTask(), GreetTask.Input.create(input.name())))
             .getOutputs()

diff --git a/flytekit-examples/src/main/java/org/flyte/examples/DynamicFibonacciWorkflowTask.java b/flytekit-examples/src/main/java/org/flyte/examples/DynamicFibonacciWorkflowTask.java
@@ -21,6 +21,7 @@
 import com.google.errorprone.annotations.Var;
 import org.flyte.examples.SumTask.SumInput;
 import org.flyte.flytekit.SdkBindingData;
+import org.flyte.flytekit.SdkBindingDataFactory;
 import org.flyte.flytekit.SdkDynamicWorkflowTask;
 import org.flyte.flytekit.SdkWorkflowBuilder;
 import org.flyte.flytekit.jackson.JacksonSdkType;
@@ -56,10 +57,10 @@ public Output run(SdkWorkflowBuilder builder, Input input) {
     if (input.n().get() < 0) {
       throw new IllegalArgumentException("n < 0");
     } else if (input.n().get() == 0) {
-      return Output.create(SdkBindingData.ofInteger(0));
+      return Output.create(SdkBindingDataFactory.of(0));
     } else {
-      @Var SdkBindingData<Long> prev = SdkBindingData.ofInteger(0);
-      @Var SdkBindingData<Long> value = SdkBindingData.ofInteger(1);
+      @Var SdkBindingData<Long> prev = SdkBindingDataFactory.of(0);
+      @Var SdkBindingData<Long> value = SdkBindingDataFactory.of(1);
       for (int i = 2; i <= input.n().get(); i++) {
         SdkBindingData<Long> next =
             builder.apply("fib-" + i, new SumTask(), SumInput.create(value, prev)).getOutputs().c();

diff --git a/flytekit-examples/src/main/java/org/flyte/examples/FibonacciLaunchPlan.java b/flytekit-examples/src/main/java/org/flyte/examples/FibonacciLaunchPlan.java
@@ -19,6 +19,7 @@
 import com.google.auto.service.AutoService;
 import com.google.auto.value.AutoValue;
 import org.flyte.flytekit.SdkBindingData;
+import org.flyte.flytekit.SdkBindingDataFactory;
 import org.flyte.flytekit.SdkLaunchPlan;
 import org.flyte.flytekit.SdkLaunchPlanRegistry;
 import org.flyte.flytekit.SimpleSdkLaunchPlanRegistry;
@@ -37,7 +38,7 @@ public FibonacciLaunchPlan() {
             .withName("FibonacciWorkflowLaunchPlan")
             .withFixedInputs(
                 JacksonSdkType.of(Input.class),
-                Input.create(SdkBindingData.ofInteger(0), SdkBindingData.ofInteger(1))));
+                Input.create(SdkBindingDataFactory.of(0), SdkBindingDataFactory.of(1))));
 
     // Register launch plan with fixed inputs specified directly
     registerLaunchPlan(

diff --git a/flytekit-examples/src/main/java/org/flyte/examples/GreetTask.java b/flytekit-examples/src/main/java/org/flyte/examples/GreetTask.java
@@ -19,6 +19,7 @@
 import com.google.auto.service.AutoService;
 import com.google.auto.value.AutoValue;
 import org.flyte.flytekit.SdkBindingData;
+import org.flyte.flytekit.SdkBindingDataFactory;
 import org.flyte.flytekit.SdkRunnableTask;
 import org.flyte.flytekit.jackson.JacksonSdkType;
 
@@ -71,6 +72,6 @@ public static Output create(SdkBindingData<String> greeting) {
   @Override
   public Output run(Input input) {
     return Output.create(
-        SdkBindingData.ofString(String.format("Welcome, %s!", input.name().get())));
+        SdkBindingDataFactory.of(String.format("Welcome, %s!", input.name().get())));
   }
 }
diff --git a/flytekit-examples/src/main/java/org/flyte/examples/NodeMetadataExampleWorkflow.java b/flytekit-examples/src/main/java/org/flyte/examples/NodeMetadataExampleWorkflow.java
@@ -20,6 +20,7 @@
 import com.google.auto.value.AutoValue;
 import java.time.Duration;
 import org.flyte.flytekit.SdkBindingData;
+import org.flyte.flytekit.SdkBindingDataFactory;
 import org.flyte.flytekit.SdkTypes;
 import org.flyte.flytekit.SdkWorkflow;
 import org.flyte.flytekit.SdkWorkflowBuilder;
@@ -50,8 +51,8 @@ public NodeMetadataExampleWorkflow() {
 
   @Override
   public Output expand(SdkWorkflowBuilder builder, Void noInput) {
-    SdkBindingData<Long> a = SdkBindingData.ofInteger(0);
-    SdkBindingData<Long> b = SdkBindingData.ofInteger(1);
+    SdkBindingData<Long> a = SdkBindingDataFactory.of(0);
+    SdkBindingData<Long> b = SdkBindingDataFactory.of(1);
 
     SdkBindingData<Long> c =
         builder

diff --git a/flytekit-examples/src/main/java/org/flyte/examples/PhoneBookWorkflow.java b/flytekit-examples/src/main/java/org/flyte/examples/PhoneBookWorkflow.java
@@ -23,6 +23,7 @@
 import java.util.List;
 import java.util.Map;
 import org.flyte.flytekit.SdkBindingData;
+import org.flyte.flytekit.SdkBindingDataFactory;
 import org.flyte.flytekit.SdkTypes;
 import org.flyte.flytekit.SdkWorkflow;
 import org.flyte.flytekit.SdkWorkflowBuilder;
@@ -61,9 +62,9 @@ public PhoneBookWorkflow() {
 
   @Override
   public Output expand(SdkWorkflowBuilder builder, Void noInput) {
-    SdkBindingData<Map<String, String>> phoneBook = SdkBindingData.ofStringMap(PHONE_BOOK);
+    SdkBindingData<Map<String, String>> phoneBook = SdkBindingDataFactory.ofStringMap(PHONE_BOOK);
 
-    SdkBindingData<List<String>> searchKeys = SdkBindingData.ofStringCollection(NAMES);
+    SdkBindingData<List<String>> searchKeys = SdkBindingDataFactory.ofStringCollection(NAMES);
 
     SdkBindingData<List<String>> phoneNumbers =
         builder

diff --git a/flytekit-examples/src/main/java/org/flyte/examples/SumTask.java b/flytekit-examples/src/main/java/org/flyte/examples/SumTask.java
@@ -19,6 +19,7 @@
 import com.google.auto.service.AutoService;
 import com.google.auto.value.AutoValue;
 import org.flyte.flytekit.SdkBindingData;
+import org.flyte.flytekit.SdkBindingDataFactory;
 import org.flyte.flytekit.SdkRunnableTask;
 import org.flyte.flytekit.jackson.Description;
 import org.flyte.flytekit.jackson.JacksonSdkType;
@@ -55,7 +56,7 @@ public static SumOutput create(SdkBindingData<Long> c) {
 
   @Override
   public SumOutput run(SumInput input) {
-    return SumOutput.create(SdkBindingData.ofInteger(input.a().get() + input.b().get()));
+    return SumOutput.create(SdkBindingDataFactory.of(input.a().get() + input.b().get()));
   }
 
   @Override