From b617ef428c02e1c1074750f63b1df47ed690fb6a Mon Sep 17 00:00:00 2001 From: Philippe Moussalli Date: Wed, 23 Aug 2023 10:58:38 +0200 Subject: [PATCH] Modify kfp command (#378) PR that modifies the KFP command to contain the new fondant cli command used to execute the component --- src/fondant/component_spec.py | 5 +-- .../example_1/kubeflow_pipeline.yml | 33 ++++++++++--------- .../example_2/kubeflow_pipeline.yml | 20 ++++++----- .../compiled_pipeline/kubeflow_pipeline.yml | 11 ++++--- .../component_specs/kubeflow_component.yaml | 5 +-- 5 files changed, 41 insertions(+), 33 deletions(-) diff --git a/src/fondant/component_spec.py b/src/fondant/component_spec.py index c00b39683..861a1ef15 100644 --- a/src/fondant/component_spec.py +++ b/src/fondant/component_spec.py @@ -284,8 +284,9 @@ def from_fondant_component_spec( "container": { "image": fondant_component.image, "command": [ - "python3", - "main.py", + "fondant", + "execute", + "main", "--input_manifest_path", {"inputPath": "input_manifest_path"}, "--metadata", diff --git a/tests/example_pipelines/compiled_pipeline/example_1/kubeflow_pipeline.yml b/tests/example_pipelines/compiled_pipeline/example_1/kubeflow_pipeline.yml index 5ddb02e38..8955b64b9 100644 --- a/tests/example_pipelines/compiled_pipeline/example_1/kubeflow_pipeline.yml +++ b/tests/example_pipelines/compiled_pipeline/example_1/kubeflow_pipeline.yml @@ -18,8 +18,9 @@ spec: - container: args: [] command: - - python3 - - main.py + - fondant + - execute + - main - --input_manifest_path - /tmp/inputs/input_manifest_path/data - --metadata @@ -54,10 +55,10 @@ spec: "{\"base_path\": \"/foo/bar\", \"pipeline_name\": \"test_pipeline\", \"run_id\": \"test_pipeline-20230101000000\", \"component_id\": \"first_component\"}", "storage_args": "a dummy string arg"}' - pipelines.kubeflow.org/component_ref: '{"digest": "2a304ce49a15404ba50dfd8b56ec43fa8ac8c29f80579d1c8fb974d3f1a5c87f"}' + pipelines.kubeflow.org/component_ref: '{"digest": "c53791e5eba77643348ea14b01bc20f273c32d827f5f1b1b896ef6965fd12d82"}' pipelines.kubeflow.org/component_spec: '{"description": "This is an example - component", "implementation": {"container": {"command": ["python3", "main.py", - "--input_manifest_path", {"inputPath": "input_manifest_path"}, "--metadata", + component", "implementation": {"container": {"command": ["fondant", "execute", + "main", "--input_manifest_path", {"inputPath": "input_manifest_path"}, "--metadata", {"inputValue": "metadata"}, "--component_spec", {"inputValue": "component_spec"}, "--input_partition_rows", {"inputValue": "input_partition_rows"}, "--storage_args", {"inputValue": "storage_args"}, "--output_manifest_path", {"outputPath": @@ -84,8 +85,9 @@ spec: - container: args: [] command: - - python3 - - main.py + - fondant + - execute + - main - --input_manifest_path - /tmp/inputs/input_manifest_path/data - --metadata @@ -120,10 +122,10 @@ spec: \"pipeline_name\": \"test_pipeline\", \"run_id\": \"test_pipeline-20230101000000\", \"component_id\": \"second_component\"}", "storage_args": "a dummy string arg"}' - pipelines.kubeflow.org/component_ref: '{"digest": "a02b0189397a2d9318982201f020dbbbe3962427ed150fe58cc69ff508cc68bb"}' + pipelines.kubeflow.org/component_ref: '{"digest": "455aeccd323115d9caae33621d3ecf5ad4de86da321f97c3761f77bc962f7fc2"}' pipelines.kubeflow.org/component_spec: '{"description": "This is an example - component", "implementation": {"container": {"command": ["python3", "main.py", - "--input_manifest_path", {"inputPath": "input_manifest_path"}, "--metadata", + component", "implementation": {"container": {"command": ["fondant", "execute", + "main", "--input_manifest_path", {"inputPath": "input_manifest_path"}, "--metadata", {"inputValue": "metadata"}, "--component_spec", {"inputValue": "component_spec"}, "--input_partition_rows", {"inputValue": "input_partition_rows"}, "--storage_args", {"inputValue": "storage_args"}, "--output_manifest_path", {"outputPath": @@ -171,8 +173,9 @@ spec: - container: args: [] command: - - python3 - - main.py + - fondant + - execute + - main - --input_manifest_path - /tmp/inputs/input_manifest_path/data - --metadata @@ -210,10 +213,10 @@ spec: "None", "metadata": "{\"base_path\": \"/foo/bar\", \"pipeline_name\": \"test_pipeline\", \"run_id\": \"test_pipeline-20230101000000\", \"component_id\": \"third_component\"}", "storage_args": "a dummy string arg"}' - pipelines.kubeflow.org/component_ref: '{"digest": "698791c6aa2ed14d4b337840116a7a995f403e5be414389b05ccf7942b9e4437"}' + pipelines.kubeflow.org/component_ref: '{"digest": "4e728e3a6242c68816de163eb5ec0398940c5fb6746adf57223ca595103e6c2a"}' pipelines.kubeflow.org/component_spec: '{"description": "This is an example - component", "implementation": {"container": {"command": ["python3", "main.py", - "--input_manifest_path", {"inputPath": "input_manifest_path"}, "--metadata", + component", "implementation": {"container": {"command": ["fondant", "execute", + "main", "--input_manifest_path", {"inputPath": "input_manifest_path"}, "--metadata", {"inputValue": "metadata"}, "--component_spec", {"inputValue": "component_spec"}, "--input_partition_rows", {"inputValue": "input_partition_rows"}, "--storage_args", {"inputValue": "storage_args"}, "--output_manifest_path", {"outputPath": diff --git a/tests/example_pipelines/compiled_pipeline/example_2/kubeflow_pipeline.yml b/tests/example_pipelines/compiled_pipeline/example_2/kubeflow_pipeline.yml index adcfdf703..9ecf9d6c0 100644 --- a/tests/example_pipelines/compiled_pipeline/example_2/kubeflow_pipeline.yml +++ b/tests/example_pipelines/compiled_pipeline/example_2/kubeflow_pipeline.yml @@ -18,8 +18,9 @@ spec: - container: args: [] command: - - python3 - - main.py + - fondant + - execute + - main - --input_manifest_path - /tmp/inputs/input_manifest_path/data - --metadata @@ -54,10 +55,10 @@ spec: "{\"base_path\": \"/foo/bar\", \"pipeline_name\": \"test_pipeline\", \"run_id\": \"test_pipeline-20230101000000\", \"component_id\": \"first_component\"}", "storage_args": "a dummy string arg"}' - pipelines.kubeflow.org/component_ref: '{"digest": "2a304ce49a15404ba50dfd8b56ec43fa8ac8c29f80579d1c8fb974d3f1a5c87f"}' + pipelines.kubeflow.org/component_ref: '{"digest": "c53791e5eba77643348ea14b01bc20f273c32d827f5f1b1b896ef6965fd12d82"}' pipelines.kubeflow.org/component_spec: '{"description": "This is an example - component", "implementation": {"container": {"command": ["python3", "main.py", - "--input_manifest_path", {"inputPath": "input_manifest_path"}, "--metadata", + component", "implementation": {"container": {"command": ["fondant", "execute", + "main", "--input_manifest_path", {"inputPath": "input_manifest_path"}, "--metadata", {"inputValue": "metadata"}, "--component_spec", {"inputValue": "component_spec"}, "--input_partition_rows", {"inputValue": "input_partition_rows"}, "--storage_args", {"inputValue": "storage_args"}, "--output_manifest_path", {"outputPath": @@ -84,8 +85,9 @@ spec: - container: args: [] command: - - python3 - - main.py + - fondant + - execute + - main - --input_manifest_path - /tmp/inputs/input_manifest_path/data - --metadata @@ -132,10 +134,10 @@ spec: "cropping_threshold": "0", "input_partition_rows": "None", "metadata": "{\"base_path\": \"/foo/bar\", \"pipeline_name\": \"test_pipeline\", \"run_id\": \"test_pipeline-20230101000000\", \"component_id\": \"image_cropping\"}", "padding": "0"}' - pipelines.kubeflow.org/component_ref: '{"digest": "e86f02b6b9cc878b6187e44bb3caf9291c3ce42c1939e19b0a97dacdc78a9d72"}' + pipelines.kubeflow.org/component_ref: '{"digest": "8a3c2b5736cf8297ad5848ec043987aed42c6fb12e6e26db25b922467b4d2d7f"}' pipelines.kubeflow.org/component_spec: '{"description": "Component that removes single-colored borders around images and crops them appropriately", "implementation": - {"container": {"command": ["python3", "main.py", "--input_manifest_path", + {"container": {"command": ["fondant", "execute", "main", "--input_manifest_path", {"inputPath": "input_manifest_path"}, "--metadata", {"inputValue": "metadata"}, "--component_spec", {"inputValue": "component_spec"}, "--input_partition_rows", {"inputValue": "input_partition_rows"}, "--cropping_threshold", {"inputValue": diff --git a/tests/example_pipelines/compiled_pipeline/kubeflow_pipeline.yml b/tests/example_pipelines/compiled_pipeline/kubeflow_pipeline.yml index 8450fe823..994a34f02 100644 --- a/tests/example_pipelines/compiled_pipeline/kubeflow_pipeline.yml +++ b/tests/example_pipelines/compiled_pipeline/kubeflow_pipeline.yml @@ -18,8 +18,9 @@ spec: - container: args: [] command: - - python3 - - main.py + - fondant + - execute + - main - --input_manifest_path - /tmp/inputs/input_manifest_path/data - --metadata @@ -57,10 +58,10 @@ spec: "{\"base_path\": \"/foo/bar\", \"pipeline_name\": \"test_pipeline\", \"run_id\": \"test_pipeline-20230101000000\", \"component_id\": \"first_component\"}", "storage_args": "a dummy string arg"}' - pipelines.kubeflow.org/component_ref: '{"digest": "2a304ce49a15404ba50dfd8b56ec43fa8ac8c29f80579d1c8fb974d3f1a5c87f"}' + pipelines.kubeflow.org/component_ref: '{"digest": "c53791e5eba77643348ea14b01bc20f273c32d827f5f1b1b896ef6965fd12d82"}' pipelines.kubeflow.org/component_spec: '{"description": "This is an example - component", "implementation": {"container": {"command": ["python3", "main.py", - "--input_manifest_path", {"inputPath": "input_manifest_path"}, "--metadata", + component", "implementation": {"container": {"command": ["fondant", "execute", + "main", "--input_manifest_path", {"inputPath": "input_manifest_path"}, "--metadata", {"inputValue": "metadata"}, "--component_spec", {"inputValue": "component_spec"}, "--input_partition_rows", {"inputValue": "input_partition_rows"}, "--storage_args", {"inputValue": "storage_args"}, "--output_manifest_path", {"outputPath": diff --git a/tests/example_specs/component_specs/kubeflow_component.yaml b/tests/example_specs/component_specs/kubeflow_component.yaml index b00b0bd19..fcf1fa866 100644 --- a/tests/example_specs/component_specs/kubeflow_component.yaml +++ b/tests/example_specs/component_specs/kubeflow_component.yaml @@ -27,8 +27,9 @@ implementation: container: image: example_component:latest command: - - python3 - - main.py + - fondant + - execute + - main - --input_manifest_path - inputPath: input_manifest_path - --metadata