Imporved docs (#144)

* fix: minor bugs with on faliure behaviours

* fix: minor bugs with on faliure behaviours

* fix: adding secrets to sdk

* docs: adding more examples

* docs: still working through

* docs: adding more examples

* fix: minor changes to API

* fix: notebook working with map parameters

* docs: more examples

* docs: more examples

* chore: gitignore

* chore: gitignore

* fix: Bug in SDK with nested nodes

* fix: Bug in argo with failure nodes

* docs: Added patching

* fix: Bug with mocked and retry implementation

* fix: disable posthog

.gitignore

@@ -6,6 +6,9 @@ __pycache__/
 # C extensions
 *.so
 
+# examples run time
+*_out*
+
 # Distribution / packaging
 .Python
 build/

examples/01-tasks/notebook.py

@@ -16,7 +16,7 @@
     │       └── simple_notebook_out.ipynb
     └── notebook.execution.log
 
-The notebook simple_notebook_out.ipynb has the captured stdout of "Hello World!".
+The notebook simple_notebook_<step name>_out.ipynb has the captured stdout of "Hello World!".
 """
 
 from runnable import NotebookTask, Pipeline

examples/01-tasks/notebook.yaml

@@ -17,7 +17,7 @@ dag:
         │       └── simple_notebook_out.ipynb
         └── notebook.execution.log
 
-    The notebook simple_notebook_out.ipynb has the captured stdout of "Hello World!".
+    The notebook simple_notebook_<step_name>_out.ipynb has the captured stdout of "Hello World!".
 
     You can run this pipeline as:
       runnable execute -f examples/01-tasks/notebook.yaml

examples/02-sequential/default_fail.py

@@ -8,7 +8,8 @@
   step 1 >> step 2 >> fail
 
 
-You can run this example by: python examples/02-sequential/default_fail.py
+You can run this example by:
+  python examples/02-sequential/default_fail.py
 """
 
 from examples.common.functions import raise_ex

examples/02-sequential/default_fail.yaml

@@ -6,7 +6,8 @@ dag:
 
     The default behavior is to traverse to step type fail and mark the run as failed.
 
-    You can run this pipeline by: runnable execute -f examples/02-sequential/default_fail.yaml
+    You can run this pipeline by:
+      runnable execute -f examples/02-sequential/default_fail.yaml
   start_at: step 1
   steps:
     step 1:

examples/02-sequential/on_failure_fail.py

@@ -12,7 +12,8 @@
 This pattern is handy when you need to do something before eventually
 failing (eg: sending a notification, updating status, etc...)
 
-Run this pipeline as: python examples/02-sequential/on_failure_fail.py
+Run this pipeline as:
+    python examples/02-sequential/on_failure_fail.py
 """
 
 from examples.common.functions import raise_ex

examples/02-sequential/on_failure_fail.yaml

@@ -12,6 +12,9 @@ dag:
 
     This pattern is handy when you need to do something before eventually
     failing (eg: sending a notification, updating status, etc...)
+
+    Run this pipeline as:
+      runnable execute -f examples/02-sequential/default_fail.yaml
   start_at: step_1
   steps:
     step_1:

examples/02-sequential/on_failure_succeed.py

@@ -12,7 +12,8 @@
 This pattern is handy when you are expecting a failure of a step
 and have ways to handle it.
 
-Run this pipeline: python examples/02-sequential/on_failure_succeed.py
+Run this pipeline:
+    python examples/02-sequential/on_failure_succeed.py
 """
 
 from examples.common.functions import raise_ex

examples/02-sequential/on_failure_succeed.yaml

@@ -31,7 +31,7 @@ dag:
       next: success
     step_4:
       type: stub
-      next: fail
+      next: success
     success:
       type: success
     fail:

examples/03-parameters/passing_parameters_notebook.py

@@ -1,3 +1,18 @@
+"""
+Demonstrates passing parameters to and from a notebook.
+
+runnable can extract JSON serializable types, pydantic models, objects from notebook.
+eg: write_parameters_from_notebook
+
+But can only inject JSON type parameters to a notebook.
+eg: read_parameters_in_notebook
+pydantic parameters are injected as dictionary.
+
+Run the below example as:
+    python examples/03-parameters/passing_parameters_notebook.py
+
+"""
+
 from examples.common.functions import read_parameter
 from runnable import NotebookTask, Pipeline, PythonTask, metric, pickled
 

examples/03-parameters/passing_parameters_notebook.yaml

@@ -0,0 +1,41 @@
+dag:
+  description: |
+    Demonstrates passing parameters to and from a notebook.
+
+    runnable can extract JSON serializable types, pydantic models, objects from notebook.
+    eg: write_parameters_from_notebook
+
+    But can only inject JSON type parameters to a notebook.
+    eg: read_parameters_in_notebook
+    pydantic parameters are injected as dictionary.
+
+    Run the below example as:
+        runnable execute examples/03-parameters/passing_parameters_notebook.yaml
+  start_at: write_parameters_from_notebook
+  steps:
+    write_parameters_from_notebook:
+      type: task
+      command_type: notebook
+      command: examples/common/write_parameters.ipynb
+      returns:
+        - name: df
+          kind: object
+        - name: integer
+        - name: floater
+        - name: stringer
+        - name: pydantic_param
+        - name: score
+      next: read_parameters
+    read_parameters:
+      type: task
+      command: examples.common.functions.read_parameter
+      next: read_parameters_in_notebook
+    read_parameters_in_notebook:
+      type: task
+      command_type: notebook
+      command: examples/common/read_parameters.ipynb
+      next: success
+    success:
+      type: success
+    fail:
+      type: fail

examples/03-parameters/passing_parameters_python.py

@@ -3,7 +3,7 @@
 tasks of the pipeline.
 
 The function, set_parameter, returns
-    - simple python data types (int, float, str)
+    - JSON serializable types
     - pydantic models
     - pandas dataframe, any "object" type
 
@@ -13,6 +13,9 @@
 Use pickled even for python data types is advised for
 reasonably large collections.
 
+Run the below example as:
+    python examples/03-parameters/passing_parameters_python.py
+
 """
 
 from examples.common.functions import read_parameter, write_parameter

examples/03-parameters/passing_parameters_python.yaml

@@ -0,0 +1,41 @@
+dag:
+  description: |
+    The below example shows how to set/get parameters in python
+    tasks of the pipeline.
+
+    The function, set_parameter, returns
+        - JSON serializable
+        - pydantic models
+        - pandas dataframe, any "object" type
+
+    pydantic models are implicitly handled by runnable
+    but "object" types should be marked as "pickled".
+
+    Use pickled even for python data types is advised for
+    reasonably large collections.
+
+    Run the pipeline as:
+      runnable execute -f examples/03-parameters/passing_parameters_python.yaml
+  start_at: write_parameters
+  steps:
+    write_parameters:
+      type: task
+      command: examples.common.functions.write_parameter
+      returns:
+        - name: df
+          kind: object
+        - name: integer
+        - name: floater
+        - name: stringer
+        - name: pydantic_param
+        - name: score
+
+      next: read_parameters
+    read_parameters:
+      type: task
+      command: examples.common.functions.read_parameter
+      next: success
+    success:
+      type: success
+    fail:
+      type: fail

examples/03-parameters/passing_parameters_shell.py

@@ -1,3 +1,18 @@
+"""
+Demonstrates passing parameters to and from shell scripts.
+
+We can extract only JSON serializable parameters from shell scripts.
+eg: write_parameters_in_shell
+
+We can only read json style parameters from shell scripts.
+eg: read_parameters_in_shell
+pydantic parameters are injected as json.
+
+Run the below example as:
+    python examples/03-parameters/passing_parameters_shell.py
+
+"""
+
 from examples.common.functions import read_unpickled_parameter
 from runnable import Pipeline, PythonTask, ShellTask, metric
 
@@ -25,11 +40,28 @@ def main():
     read_parameters = PythonTask(
         function=read_unpickled_parameter,
         name="read_parameters",
+    )
+
+    read_parameters_command = """
+    if [ "$integer" = 1 ] \
+        && [ "$floater" = 3.14 ] \
+        && [ "$stringer" = "hello" ] \
+        && [ "$pydantic_param" = '{"x": 10, "foo": "bar"}' ]; then
+            echo "yaay"
+            exit 0;
+        else
+            echo "naay"
+            exit 1;
+    fi
+    """
+    read_parameters_in_shell = ShellTask(
+        name="read_parameters_in_shell",
+        command=read_parameters_command,
         terminate_with_success=True,
     )
 
     pipeline = Pipeline(
-        steps=[write_parameters_in_shell, read_parameters],
+        steps=[write_parameters_in_shell, read_parameters, read_parameters_in_shell],
     )
 
     _ = pipeline.execute()

examples/03-parameters/passing_parameters_shell.yaml

@@ -0,0 +1,55 @@
+dag:
+  description: |
+    Demonstrates passing parameters to and from shell scripts.
+
+    We can extract only json style parameters from shell scripts.
+    eg: write_parameters_in_shell
+
+    We can only read json style parameters from shell scripts.
+    eg: read_parameters_in_shell
+    pydantic parameters are injected as json.
+
+    Run the pipeline as:
+      runnable execute -f examples/03-parameters/passing_parameters_shell.yaml
+
+  start_at: write_parameters_in_shell
+  steps:
+    write_parameters_in_shell:
+      type: task
+      command_type: shell
+      command: |
+        export integer=1
+        export floater=3.14
+        export stringer="hello"
+        export pydantic_param='{"x": 10, "foo": "bar"}'
+        export score=0.9
+      returns:
+        - name: integer
+        - name: floater
+        - name: stringer
+        - name: pydantic_param
+        - name: score
+      next: read_parameters
+    read_parameters:
+      type: task
+      command: examples.common.functions.read_unpickled_parameter
+      next: read_parameters_in_shell
+    read_parameters_in_shell:
+      type: task
+      command_type: shell
+      command: |
+        if [ "$integer" = 1 ] \
+          && [ "$floater" = 3.14 ] \
+          && [ "$stringer" = "hello" ] \
+          && [ "$pydantic_param" = '{"x": 10, "foo": "bar"}' ]; then
+              echo "yaay"
+              exit 0;
+        else
+              echo "naay"
+              exit 1;
+        fi
+      next: success
+    success:
+      type: success
+    fail:
+      type: fail

examples/03-parameters/static_parameters_non_python.py

@@ -11,8 +11,14 @@
         foo: bar
 
 runnable exposes the nested parameters as dictionary for notebook based tasks
-as a json string for the shell based tasks.
+and as a json string for the shell based tasks.
 
+You can set the initial parameters from environment variables as well.
+eg: Any environment variable prefixed by "RUNNABLE_PRM_" will be picked up by runnable
+
+
+Run this pipeline as:
+    python examples/03-parameters/static_parameters_non_python.py
 """
 
 from runnable import NotebookTask, Pipeline, ShellTask

examples/03-parameters/static_parameters_non_python.yaml

@@ -12,7 +12,16 @@ dag:
           foo: bar
 
     runnable exposes the nested parameters as dictionary for notebook based tasks
-    as a json string for the shell based tasks.
+    and as a json string for the shell based tasks.
+
+    You can set the initial parameters from environment variables as well.
+    eg: Any environment variable prefixed by "RUNNABLE_PRM_" will be picked up by runnable
+
+
+    Run this pipeline as:
+      runnable execute -f 03-parameters/static_parameters_non_python.yaml \
+        -p common/initial_parameters.yaml
+
   start_at: read_params_in_notebook
   steps:
     read_params_in_notebook:

examples/03-parameters/static_parameters_python.py

@@ -14,8 +14,16 @@
 If no annotation is provided, the parameter is assumed to be a dictionary.
 eg: read_initial_params_as_json
 
+You can set the initial parameters from environment variables as well.
+eg: Any environment variable prefixed by "RUNNABLE_PRM_" will be picked up by runnable
+
+Run this pipeline as:
+    python examples/03-parameters/static_parameters_python.py
+
 """
 
+import os
+
 from examples.common.functions import (
     read_initial_params_as_json,
     read_initial_params_as_pydantic,
@@ -46,4 +54,6 @@ def main():
 
 
 if __name__ == "__main__":
+    # Any parameter prefixed by "RUNNABLE_PRM_" will be picked up by runnable
+    os.environ["RUNNABLE_PRM_envvar"] = "from env"
     main()

examples/03-parameters/static_parameters_python.yaml

@@ -14,6 +14,13 @@ dag:
 
     If no annotation is provided, the parameter is assumed to be a dictionary.
     eg: read_initial_params_as_json
+
+    You can set the initial parameters from environment variables as well.
+    eg: Any environment variable prefixed by "RUNNABLE_PRM_" will be picked up by runnable
+
+    Run this pipeline by:
+      runnable execute -f 03-parameters/static_parameters_python.yaml \
+        -p examples/common/initial_parameters.yaml
   start_at: read_params_as_pydantic
   steps:
     read_params_as_pydantic: