Skip to content

Commit

Permalink
Sbachmei/mic 5560/choice sections (#125)
Browse files Browse the repository at this point in the history
  • Loading branch information
@stevebachmeier
stevebachmeier authored Dec 10, 2024
1 parent ff737ba commit 128b14c
Show file tree
Hide file tree
Showing 16 changed files with 968 additions and 184 deletions.
29 changes: 28 additions & 1 deletion src/easylink/implementation_metadata.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,24 @@ step_4_python_pandas:
INPUT_ENV_VARS: "DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS"
outputs:
step_4_main_output: result.parquet
step_5_python_pandas:
steps:
- step_5
image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
script_cmd: python /dummy_step.py
env:
INPUT_ENV_VARS: "DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS"
outputs:
step_5_main_output: result.parquet
step_6_python_pandas:
steps:
- step_6
image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
script_cmd: python /dummy_step.py
env:
INPUT_ENV_VARS: "DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS"
outputs:
step_6_main_output: result.parquet
step_4a_python_pandas:
steps:
- step_4a
Expand All @@ -39,6 +57,15 @@ step_4a_python_pandas:
outputs:
step_4a_main_output: result.parquet
step_4b_python_pandas:
steps:
- step_4b
image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/r-image.sif
script_cmd: Rscript /dummy_step.R
env:
INPUT_ENV_VARS: "DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS,DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS"
outputs:
step_4b_main_output: result.parquet
step_4b_r:
steps:
- step_4b
image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
Expand Down Expand Up @@ -139,4 +166,4 @@ step_3_and_step_4_combined_python_pandas:
image_path: /mnt/team/simulation_science/priv/engineering/er_ecosystem/images/python_pandas.sif
script_cmd: python /dummy_step.py
outputs:
step_4_main_output: result.parquet
step_4_main_output: result.parquet
233 changes: 162 additions & 71 deletions src/easylink/pipeline_schema_constants/development.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
OutputSlotMapping,
)
from easylink.step import (
ChoiceStep,
HierarchicalStep,
InputStep,
LoopStep,
Expand Down Expand Up @@ -62,87 +63,177 @@
)
],
),
HierarchicalStep(
step_name="step_4",
ChoiceStep(
step_name="choice_section",
input_slots=[
InputSlot(
name="step_4_main_input",
name="choice_section_main_input",
env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
validator=validate_input_file_dummy,
),
InputSlot(
name="step_4_secondary_input",
name="choice_section_secondary_input",
env_var="DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS",
validator=validate_input_file_dummy,
),
],
output_slots=[OutputSlot("step_4_main_output")],
nodes=[
Step(
step_name="step_4a",
input_slots=[
InputSlot(
name="step_4a_main_input",
env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
validator=validate_input_file_dummy,
output_slots=[OutputSlot("choice_section_main_output")],
choices={
"simple": {
"nodes": [
HierarchicalStep(
step_name="step_4",
input_slots=[
InputSlot(
name="step_4_main_input",
env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
validator=validate_input_file_dummy,
),
InputSlot(
name="step_4_secondary_input",
env_var="DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS",
validator=validate_input_file_dummy,
),
],
output_slots=[OutputSlot("step_4_main_output")],
nodes=[
Step(
step_name="step_4a",
input_slots=[
InputSlot(
name="step_4a_main_input",
env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
validator=validate_input_file_dummy,
),
InputSlot(
name="step_4a_secondary_input",
env_var="DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS",
validator=validate_input_file_dummy,
),
],
output_slots=[OutputSlot("step_4a_main_output")],
),
Step(
step_name="step_4b",
input_slots=[
InputSlot(
name="step_4b_main_input",
env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
validator=validate_input_file_dummy,
),
InputSlot(
name="step_4b_secondary_input",
env_var="DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS",
validator=validate_input_file_dummy,
),
],
output_slots=[OutputSlot("step_4b_main_output")],
),
],
edges=[
EdgeParams(
source_node="step_4a",
target_node="step_4b",
output_slot="step_4a_main_output",
input_slot="step_4b_main_input",
),
],
input_slot_mappings=[
InputSlotMapping(
parent_slot="step_4_main_input",
child_node="step_4a",
child_slot="step_4a_main_input",
),
InputSlotMapping(
parent_slot="step_4_secondary_input",
child_node="step_4a",
child_slot="step_4a_secondary_input",
),
InputSlotMapping(
parent_slot="step_4_secondary_input",
child_node="step_4b",
child_slot="step_4b_secondary_input",
),
],
output_slot_mappings=[
OutputSlotMapping(
parent_slot="step_4_main_output",
child_node="step_4b",
child_slot="step_4b_main_output",
),
],
),
InputSlot(
name="step_4a_secondary_input",
env_var="DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS",
validator=validate_input_file_dummy,
],
"edges": [],
"input_slot_mappings": [
InputSlotMapping(
parent_slot="choice_section_main_input",
child_node="step_4",
child_slot="step_4_main_input",
),
InputSlotMapping(
parent_slot="choice_section_secondary_input",
child_node="step_4",
child_slot="step_4_secondary_input",
),
],
output_slots=[OutputSlot("step_4a_main_output")],
),
Step(
step_name="step_4b",
input_slots=[
InputSlot(
name="step_4b_main_input",
env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
validator=validate_input_file_dummy,
"output_slot_mappings": [
OutputSlotMapping(
parent_slot="choice_section_main_output",
child_node="step_4",
child_slot="step_4_main_output",
),
InputSlot(
name="step_4b_secondary_input",
env_var="DUMMY_CONTAINER_SECONDARY_INPUT_FILE_PATHS",
validator=validate_input_file_dummy,
],
},
"complex": {
"nodes": [
Step(
step_name="step_5",
input_slots=[
InputSlot(
name="step_5_main_input",
env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
validator=validate_input_file_dummy,
),
],
output_slots=[OutputSlot("step_5_main_output")],
),
Step(
step_name="step_6",
input_slots=[
InputSlot(
name="step_6_main_input",
env_var="DUMMY_CONTAINER_MAIN_INPUT_FILE_PATHS",
validator=validate_input_file_dummy,
),
],
output_slots=[OutputSlot("step_6_main_output")],
),
],
output_slots=[OutputSlot("step_4b_main_output")],
),
],
edges=[
EdgeParams(
source_node="step_4a",
target_node="step_4b",
output_slot="step_4a_main_output",
input_slot="step_4b_main_input",
),
],
input_slot_mappings=[
InputSlotMapping(
parent_slot="step_4_main_input",
child_node="step_4a",
child_slot="step_4a_main_input",
),
InputSlotMapping(
parent_slot="step_4_secondary_input",
child_node="step_4a",
child_slot="step_4a_secondary_input",
),
InputSlotMapping(
parent_slot="step_4_secondary_input",
child_node="step_4b",
child_slot="step_4b_secondary_input",
),
],
output_slot_mappings=[
OutputSlotMapping(
parent_slot="step_4_main_output",
child_node="step_4b",
child_slot="step_4b_main_output",
),
],
"edges": [
EdgeParams(
source_node="step_5",
target_node="step_6",
output_slot="step_5_main_output",
input_slot="step_6_main_input",
),
],
"input_slot_mappings": [
InputSlotMapping(
parent_slot="choice_section_main_input",
child_node="step_5",
child_slot="step_5_main_input",
),
],
"output_slot_mappings": [
OutputSlotMapping(
parent_slot="choice_section_main_output",
child_node="step_6",
child_slot="step_6_main_output",
),
],
},
},
),
OutputStep(
input_slots=[
Expand All @@ -159,9 +250,9 @@
),
EdgeParams(
source_node="input_data",
target_node="step_4",
target_node="choice_section",
output_slot="all",
input_slot="step_4_secondary_input",
input_slot="choice_section_secondary_input",
),
EdgeParams(
source_node="step_1",
Expand All @@ -177,14 +268,14 @@
),
EdgeParams(
source_node="step_3",
target_node="step_4",
target_node="choice_section",
output_slot="step_3_main_output",
input_slot="step_4_main_input",
input_slot="choice_section_main_input",
),
EdgeParams(
source_node="step_4",
source_node="choice_section",
target_node="results",
output_slot="step_4_main_output",
output_slot="choice_section_main_output",
input_slot="result",
),
]
Expand Down
Loading

0 comments on commit 128b14c

Please sign in to comment.