Merge pull request #33 from metno/rework-plugins

Rework how functions from edr, ogcapi and profile is collected and ru…

README.md

@@ -4,7 +4,7 @@
 
 ## What is sedr?
 
-An experimental validator for OGC EDR APIs using schemathesis. Main focus will be on the Rodeo Profile, which is a subset of the OGC EDR API.
+An experimental validator for OGC EDR APIs. Main focus will be on the Rodeo Profile, which is a subset of the OGC EDR API.
 
 ## Who is responsible?
 
@@ -40,17 +40,28 @@ Run manually as noted in [Test it out](#test-it-out), or add it to your CI using
 
 ## Overview of architecture
 
+- __init__ includes tests from ogcapi, edrreq and rodeoprofile at startup. Tests are categorized as landing, conformance and collection.
+- Landing and conformance tests are run first, in the preflight phase.
+- Then schemathesis will validate the OpenAPI spec and run lots of automatic tests, including fuzzing of query parameters. Collection tests are run during this phase.
+
 ## Documentation
 
+- Use --rodeo-profile to force a test against the profile
+- Use --strict to also fail on SHOULD requirements.
+- Use --log-file debug.log to get all output. For docker variant, see [Test it out](#test-it-out).
+
 ### Limitations
 
 - Assuming Openapi 3.1
 - Assuming OGC EDR API version 1.2 (draft)
 - Few, basic tests for now
 - Will focus more on profiles (limitations within the EDR spec) like <https://github.com/EURODEO/rodeo-edr-profile> than the full EDR spec.
-- Use --rodeo-profile to force a test against the profile (will happen automatically if conformance includes the profile)
 
-### Understanding errors
+### Testing the sedr code to look for regressions
+
+For development, source a venv and run `tox p` to run all tests.
+
+### Understanding errors from schemathesis
 
 For each "FAILED" line, you can scroll back to see the full error and, if relevant, with a curl-example to reproduce it.
 
@@ -96,7 +107,7 @@ ERROR sedr/schemat.py - schemathesis.exceptions.SchemaError: Failed to load sche
 
 #### Wrong API version / missing conformance link
 
-Sedr supports EDR 1.1, but the API is EDR 1.0.
+Sedr wants EDR 1.1, but the API is EDR 1.0.
 
 ```python
         if not requirementA2_2_A5:

requirements.txt

@@ -2,3 +2,4 @@ schemathesis~=3.34
 pytest~=8.3
 shapely~=2.0
 requests~=2.32.0
+rich~=13.8.0

sedr/__init__.py

@@ -2,12 +2,16 @@
 
 __author__ = "Lars Falk-Petersen"
 __license__ = "GPL-3.0"
-__version__ = "v0.7.9"
+__version__ = "v0.8.0"
 
 import sys
 import pytest
+
 import util
 import preflight
+import edreq12 as edreq
+import ogcapi10 as ogcapi
+import rodeoprofile10 as rodeoprofile
 
 
 def run_schemat() -> None:
@@ -18,6 +22,27 @@ def run_schemat() -> None:
 
 def main() -> None:
     """Run the main program."""
+
+    # Collect tests to run
+    util.test_functions["landing"] += edreq.tests_landing + ogcapi.tests_landing
+    util.test_functions["conformance"] += (
+        edreq.tests_conformance + ogcapi.tests_conformance
+    )
+    util.test_functions["collection"] += (
+        edreq.tests_collection + ogcapi.tests_collections
+    )
+    if util.args.rodeo_profile:
+        util.logger.info(
+            "Including tests for Rodeo profile %s", rodeoprofile.conformance_url
+        )
+        util.test_functions["landing"] += rodeoprofile.tests_landing
+        util.test_functions["conformance"] += rodeoprofile.tests_conformance
+        util.test_functions["collection"] += rodeoprofile.tests_collection
+
+    # TODO: include profile tests based on conformance_url, https://github.com/metno/sedr/issues/32
+    # if rodeoprofile.conformance_url in conformance_json["conformsTo"]:
+    #     util.args.rodeo_profile = True
+
     if preflight.main():
         run_schemat()
     else:

sedr/edreq12.py

@@ -1,5 +1,6 @@
 """EDR requirements."""
 
+from collections.abc import Callable
 import util
 
 edr_version = "1.2"
@@ -16,7 +17,7 @@
 ]
 
 
-def requirementA2_2_A5(jsondata: dict, siteurl="") -> tuple[bool, str]:
+def requirementA2_2_A5(jsondata: dict) -> tuple[bool, str]:
     """
     OGC API - Environmental Data Retrieval Standard
     Version: 1.2
@@ -26,40 +27,21 @@ def requirementA2_2_A5(jsondata: dict, siteurl="") -> tuple[bool, str]:
     jsondata should be the "conformsTo"-part of the conformance page.
     """
     spec_url = f"{edr_root_url}#req_core_conformance"
+
     if "conformsTo" not in jsondata:
-        return (
-            False,
-            f"Conformance page <{siteurl}conformance> does not contain a "
-            f"conformsTo attribute. See <{spec_url}> for more info.",
+        return False, (
+            f"Conformance page does not contain a "
+            f"conformsTo attribute. See <{spec_url}> for more info."
         )
+
     for url in conformance_urls:
         if url not in jsondata["conformsTo"]:
-            return (
-                False,
-                f"Conformance page <{siteurl}conformance> does not contain "
-                f"the core edr class {url}. See <{spec_url}> for more info.",
+            return False, (
+                f"Conformance page does not contain "
+                f"the core edr class {url}. See <{spec_url}> for more info."
             )
 
-    util.logger.debug(
-        "requirementA2_2_A5: conformance page contains the required EDR classes."
-    )
-    return True, ""
-
-
-def requirementA2_2_A7(version: int) -> tuple[bool, str]:
-    """
-    OGC API - Environmental Data Retrieval Standard
-    Version: 1.2
-    Requirement Annex A2.2 A7
-
-    Check if HTTP1.1 was used.
-    """
-    spec_url = f"{edr_root_url}#_req_core_http"
-    if version == 11:
-        util.logger.debug("requirementA2_2_A7 HTTP version 1.1 was used.")
-        return True, ""
-
-    return False, f"HTTP version 1.1 was not used. See <{spec_url}> for more info."
+    return True, "Conformance page contains the required EDR classes."
 
 
 def requirementA11_1(jsondata: dict) -> tuple[bool, str]:
@@ -77,8 +59,7 @@ def requirementA11_1(jsondata: dict) -> tuple[bool, str]:
             if (
                 "oas31" in url or "oas30" in url  # TODO: oas30 should be removed
             ):
-                util.logger.debug("requirementA11_1 Found openapi class <%s>", url)
-                return True, url
+                return True, f"Found openapi class <{url}>. "
             return (
                 False,
                 f"OpenAPI version {util.args.openapi_version} and version in "
@@ -90,3 +71,31 @@ def requirementA11_1(jsondata: dict) -> tuple[bool, str]:
         f"Conformance page /conformance does not contain an openapi class. "
         f"See <{spec_url}> for more info.",
     )
+
+
+def requrementA5_2(jsondata: dict) -> tuple[bool, str]:
+    """
+    OGC API - Environmental Data Retrieval Standard
+    Version: 1.2
+    Requirement A5.2
+
+    Check extent spatial bbox
+    """
+
+    spec_url = f"{edr_root_url}#req_core_rc-bbox-definition"
+
+    extent = None
+    extent = util.parse_spatial_bbox(jsondata)
+    if extent == [] or len(extent) > 1 or not isinstance(extent, list):
+        return (
+            False,
+            f"Extent→spatial→bbox should be a list of bboxes with exactly "
+            f"one bbox in, found {len(extent)} in collection "
+            f"<{jsondata['id']}>. See {spec_url} for more info.",
+        )
+    return True, f"Extent→spatial→bbox for collection is {extent}"
+
+
+tests_landing: list[Callable[[dict], tuple[bool, str]]] = []
+tests_conformance = [requirementA2_2_A5, requirementA11_1]
+tests_collection = [requrementA5_2]