WIP

.gitignore

@@ -3,4 +3,6 @@ __pycache__
 .coverage
 behave_debug.py
 features/fixtures/data/
-data
+data
+output
+.venv

dpypelines/pipeline/configuration.py

@@ -1,10 +1,17 @@
+from pathlib import Path
 import re
 
+from dpypelines.pipeline.csvcubed_ingress_v1 import csvcubed_ingress_v1
 from dpypelines.pipeline.dataset_ingress_v1 import dataset_ingress_v1
 from dpypelines.pipeline.generic_file_ingress_v1 import generic_file_ingress_v1
-from dpypelines.pipeline.shared.transforms.sanity_check import sdmx_sanity_check_v1
+from dpypelines.pipeline.shared.transforms.sanity_check import (
+    csv_sanity_check_v1,
+    json_sanity_check_v1,
+    sdmx_sanity_check_v1,
+)
 from dpypelines.pipeline.shared.transforms.sdmx.v20 import sdmx_compact_2_0_prototype_1
 from dpypelines.pipeline.shared.transforms.sdmx.v21 import sdmx_generic_2_1_prototype_1
+from csvcubed.cli.buildcsvw.build import build_csvw
 
 # Set a regex pattern matching the `source_id` as `CONFIGURATION` dictionary key
 # All fields are required in order for a pipeline transform to run successfully
@@ -38,6 +45,27 @@
         "supplementary_distributions": {},
         "secondary_function": generic_file_ingress_v1,
     },
+    "^.*csvcubed$": {
+        "config_version": 1,
+        "transform": build_csvw,
+        "transform_inputs": {
+            "^dataset.csv$": csv_sanity_check_v1,
+            "^metadata.json$": json_sanity_check_v1,
+        },
+        "transform_kwargs": {
+            "output_directory": Path("output"),
+            "validation_errors_file_name": "validation_errors.json",
+        },
+        "required_files": [
+            {"matches": "^dataset.csv$"},
+            {"matches": "^metadata.json$"},
+        ],
+        "supplementary_distributions": [
+            {"matches": "^dataset.csv$"},
+            {"matches": "^metadata.json$"},
+        ],
+        "secondary_function": csvcubed_ingress_v1,
+    },
 }