feat: removing explicit addition of success and fail nodes from yaml

AstraZeneca · May 13, 2024 · 7e69abb · 7e69abb
1 parent 43a42e5
commit 7e69abb
Show file tree

Hide file tree

Showing 20 changed files with 3 additions and 97 deletions.
diff --git a/examples/01-tasks/notebook.yaml b/examples/01-tasks/notebook.yaml
@@ -29,7 +29,3 @@ dag:
       command_type: notebook
       command: examples/common/simple_notebook.ipynb # The path is relative to the root of the project.
       next: success
-    success:
-      type: success
-    fail:
-      type: fail
diff --git a/examples/01-tasks/python_tasks.yaml b/examples/01-tasks/python_tasks.yaml
@@ -21,7 +21,3 @@ dag:
       type: task
       command: examples.common.functions.hello # dotted path to the function.
       next: success
-    success:
-      type: success
-    fail:
-      type: fail
diff --git a/examples/01-tasks/scripts.yaml b/examples/01-tasks/scripts.yaml
@@ -18,7 +18,3 @@ dag:
       command_type: shell
       command: echo "hello world!!" # The path is relative to the root of the project.
       next: success
-    success:
-      type: success
-    fail:
-      type: fail
diff --git a/examples/01-tasks/stub.yaml b/examples/01-tasks/stub.yaml
@@ -27,7 +27,3 @@ dag:
     step 3:
       type: stub
       next: success
-    success:
-      type: success
-    fail:
-      type: fail
diff --git a/examples/02-sequential/default_fail.yaml b/examples/02-sequential/default_fail.yaml
@@ -21,7 +21,3 @@ dag:
     step 3:
       type: stub # This will never execute
       next: success
-    success:
-      type: success
-    fail:
-      type: fail
diff --git a/examples/02-sequential/on_failure_fail.yaml b/examples/02-sequential/on_failure_fail.yaml
@@ -32,7 +32,3 @@ dag:
     step_4:
       type: stub
       next: fail
-    success:
-      type: success
-    fail:
-      type: fail
diff --git a/examples/02-sequential/on_failure_succeed.yaml b/examples/02-sequential/on_failure_succeed.yaml
@@ -32,7 +32,3 @@ dag:
     step_4:
       type: stub
       next: success
-    success:
-      type: success
-    fail:
-      type: fail
diff --git a/examples/02-sequential/traversal.yaml b/examples/02-sequential/traversal.yaml
@@ -43,8 +43,4 @@ dag:
       type: task
       command_type: notebook
       command: examples/common/simple_notebook.ipynb # The path is relative to the root of the project.
-      next: success
-    success: # (3)
-      type: success
-    fail:
-      type: fail
+      next: success # (3)
diff --git a/examples/03-parameters/passing_parameters_notebook.yaml b/examples/03-parameters/passing_parameters_notebook.yaml
@@ -36,7 +36,3 @@ dag:
       command_type: notebook
       command: examples/common/read_parameters.ipynb
       next: success
-    success:
-      type: success
-    fail:
-      type: fail
diff --git a/examples/03-parameters/passing_parameters_python.yaml b/examples/03-parameters/passing_parameters_python.yaml
@@ -36,7 +36,3 @@ dag:
       type: task
       command: examples.common.functions.read_parameter
       next: success
-    success:
-      type: success
-    fail:
-      type: fail
diff --git a/examples/03-parameters/passing_parameters_shell.yaml b/examples/03-parameters/passing_parameters_shell.yaml
@@ -49,7 +49,3 @@ dag:
               exit 1;
         fi
       next: success
-    success:
-      type: success
-    fail:
-      type: fail
diff --git a/examples/03-parameters/static_parameters_non_python.yaml b/examples/03-parameters/static_parameters_non_python.yaml
@@ -44,7 +44,3 @@ dag:
               exit 1;
         fi
       next: success
-    success:
-      type: success
-    fail:
-      type: fail
diff --git a/examples/03-parameters/static_parameters_python.yaml b/examples/03-parameters/static_parameters_python.yaml
@@ -31,7 +31,3 @@ dag:
       type: task
       command: examples.common.functions.read_initial_params_as_json
       next: success
-    success:
-      type: success
-    fail:
-      type: fail
diff --git a/examples/04-catalog/catalog.yaml b/examples/04-catalog/catalog.yaml
@@ -113,7 +113,3 @@ dag:
         rm df.csv || true && \
         rm data_folder/data.txt || true
       next: success
-    success:
-      type: success
-    fail:
-      type: fail
diff --git a/examples/06-parallel/nesting.yaml b/examples/06-parallel/nesting.yaml
@@ -24,10 +24,6 @@ branch: &simple_branch
       command_type: notebook
       command: examples/common/simple_notebook.ipynb # The path is relative to the root of the project.
       next: success
-    success:
-      type: success
-    fail:
-      type: fail
 
 
 # This branch is similar to a branch parallel.yaml
@@ -40,10 +36,6 @@ nested_branch: &nested_branch
       branches:
         branch1: *simple_branch
         branch2: *simple_branch
-    success:
-      type: success
-    failure:
-      type: fail
 
 
 # The pipeline of nested parallel branches
@@ -56,7 +48,3 @@ dag:
       branches:
         branch1: *nested_branch
         branch2: *nested_branch
-    success:
-      type: success
-    failure:
-      type: fail
diff --git a/examples/06-parallel/parallel.yaml b/examples/06-parallel/parallel.yaml
@@ -23,10 +23,6 @@ branch: &branch
       command_type: notebook
       command: examples/common/simple_notebook.ipynb # The path is relative to the root of the project.
       next: success
-    success:
-      type: success
-    fail:
-      type: fail
 
 
 dag:
@@ -47,7 +43,3 @@ dag:
       branches:
         branch1: *branch
         branch2: *branch
-    success:
-      type: success
-    failure:
-      type: fail
diff --git a/examples/07-map/custom_reducer.yaml b/examples/07-map/custom_reducer.yaml
@@ -33,10 +33,6 @@ branch: &branch
       type: task
       command: examples.common.functions.read_processed_chunk
       next: success
-    success:
-      type: success
-    fail:
-      type: fail
 
 dag:
   description: |
@@ -75,7 +71,3 @@ dag:
       type: task
       command: examples.common.functions.assert_custom_reducer
       next: success
-    success:
-      type: success
-    fail:
-      type: fail
diff --git a/examples/07-map/map.yaml b/examples/07-map/map.yaml
@@ -33,10 +33,6 @@ branch: &branch
       type: task
       command: examples.common.functions.read_processed_chunk
       next: success
-    success:
-      type: success
-    fail:
-      type: fail
 
 
 dag:
@@ -76,7 +72,3 @@ dag:
       type: task
       command: examples.common.functions.assert_default_reducer
       next: success
-    success:
-      type: success
-    fail:
-      type: fail
diff --git a/examples/09-retry/python_tasks.yaml b/examples/09-retry/python_tasks.yaml
@@ -31,7 +31,3 @@ dag:
       type: task
       command: examples.common.functions.read_initial_params_as_json
       next: success
-    success:
-      type: success
-    fail:
-      type: fail
diff --git a/runnable/graph.py b/runnable/graph.py
@@ -342,6 +342,8 @@ def create_graph(dag_config: Dict[str, Any], internal_branch_name: str = "") ->
         node = create_node(name, step_config=step_config, internal_branch_name=internal_branch_name)
         graph.add_node(node)
 
+    graph.add_terminal_nodes(internal_branch_name=internal_branch_name)
+
     graph.check_graph()
 
     return graph