From 1fa2c98218cbd3a6e1c72d17d1f7ef0899636b04 Mon Sep 17 00:00:00 2001 From: Francis Charette Migneault Date: Mon, 26 Aug 2024 18:22:30 -0400 Subject: [PATCH] fix test supporting collection input (fixes #682) - cwltool patch loadContents (relates to https://github.com/common-workflow-language/cwltool/pull/2036) --- CHANGES.rst | 7 +++++- requirements.txt | 5 ++-- .../EchoFeatures/echo_features.cwl | 11 ++++---- tests/functional/test_wps_package.py | 25 +++++++++++-------- 4 files changed, 30 insertions(+), 18 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4e28ee630..070092469 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -12,7 +12,12 @@ Changes Changes: -------- -- No change. +- Add support of *OGC API - Processes: Part 3* ``collection`` as input to a `Process` + (fixes `#682 `_). +- Update ``cwltool`` with fork + `fmigneault/cwltool @ fix-load-contents-array `_ + until ``loadContents`` behavior is resolved for ``type: File[]`` + (relates to `common-workflow-language/cwltool#2036 `_). Fixes: ------ diff --git a/requirements.txt b/requirements.txt index a203d89a4..29253898e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -35,7 +35,8 @@ cryptography # use cwltool gpu-enabled support until integrated within the original tool # (https://github.com/common-workflow-language/common-workflow-language/issues/587) ### git+https://github.com/crim-ca/cwltool@docker-gpu#egg=cwltool -cwltool==3.1.20230906142556 +##cwltool==3.1.20230906142556 +cwltool @ git+https://github.com/fmigneault/cwltool.git@fix-load-contents-array docker>=7.1 duration esgf-compute-api @ git+https://github.com/ESGF/esgf-compute-api.git@v2.3.7 @@ -58,7 +59,7 @@ lxml mako # force use of later mistune (https://github.com/common-workflow-language/schema_salad/pull/619#issuecomment-1346025607) # employed by cwltool -> schema-salad -> mistune -mistune>=2.0.3,<2.1 +#mistune>=2.0.3,<2.1 mypy_boto3_s3 numpy>=1.22.2,<2; python_version < "3.10" numpy>=1.22.2; python_version >= "3.10" diff --git a/tests/functional/application-packages/EchoFeatures/echo_features.cwl b/tests/functional/application-packages/EchoFeatures/echo_features.cwl index 16a2b2ef7..9cc630ed1 100644 --- a/tests/functional/application-packages/EchoFeatures/echo_features.cwl +++ b/tests/functional/application-packages/EchoFeatures/echo_features.cwl @@ -22,14 +22,15 @@ inputs: } return "http://www.opengis.net/def/glossary/term/FeatureCollection"; } + loadContents: true inputBinding: valueFrom: | ${ if (Array.isArray(inputs.features)) { - return { + return JSON.stringify({ "type": "FeatureCollection", - "features": inputs.features.every(item => item.contents) - }; + "features": inputs.features.map(item => JSON.parse(item.contents)) + }); } return inputs.features.contents; } @@ -38,8 +39,8 @@ outputs: type: File format: "ogc-term:FeatureCollection" outputBinding: - glob: "features.json" -stdout: "features.json" + glob: "features.geojson" +stdout: "features.geojson" $namespaces: iana: "https://www.iana.org/assignments/media-types/" ogc-term: "http://www.opengis.net/def/glossary/term/" diff --git a/tests/functional/test_wps_package.py b/tests/functional/test_wps_package.py index 9b3f83033..9489faefb 100644 --- a/tests/functional/test_wps_package.py +++ b/tests/functional/test_wps_package.py @@ -2277,13 +2277,11 @@ def test_execute_job_with_collection_input_geojson_feature_collection(self): tmp_dir = stack.enter_context(tempfile.TemporaryDirectory()) # pylint: disable=R1732 stack.enter_context(mocked_file_server(tmp_dir, tmp_host, settings=self.settings, mock_browse_index=True)) - col_file = os.path.join(tmp_dir, "test.geojson") exec_body_val = self.retrieve_payload(name, "execute", local=True) + col_file = os.path.join(tmp_dir, "test.json") + col_feats = exec_body_val["inputs"]["features"]["value"] # type: JSON with open(col_file, mode="w", encoding="utf-8") as tmp_feature_collection_geojson: - json.dump( - exec_body_val["inputs"]["features"]["value"], - tmp_feature_collection_geojson, - ) + json.dump(col_feats, tmp_feature_collection_geojson) col_exec_body = { "mode": ExecuteMode.ASYNC, @@ -2291,9 +2289,9 @@ def test_execute_job_with_collection_input_geojson_feature_collection(self): "inputs": { "features": { # accessed directly as a static GeoJSON FeatureCollection - "collection": "https://mocked-file-server.com/test.geojson", + "collection": "https://mocked-file-server.com/test.json", "format": ExecuteCollectionFormat.GEOJSON, - "type": ContentType.APP_GEOJSON, + "schema": "http://www.opengis.net/def/glossary/term/FeatureCollection", }, } } @@ -2307,9 +2305,16 @@ def test_execute_job_with_collection_input_geojson_feature_collection(self): status_url = resp.json["location"] results = self.monitor_job(status_url) - assert "outputs" in results - - raise NotImplementedError # FIXME: implement! (see above bbox case for inspiration) + assert "features" in results + + job_id = status_url.rsplit("/", 1)[-1] + wps_dir = get_wps_output_dir(self.settings) + job_dir = os.path.join(wps_dir, job_id) + job_out = os.path.join(job_dir, "features", "features.geojson") + assert os.path.isfile(job_out), f"Invalid output file not found: [{job_out}]" + with open(job_out, mode="r", encoding="utf-8") as out_fd: + out_data = json.load(out_fd) + assert out_data["features"] == col_feats["features"] def test_execute_job_with_collection_input_ogc_features(self): name = "EchoFeatures"