diff --git a/README.md b/README.md index 47716d6..d5031b3 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ ## What is sedr? -An experimental validator for OGC EDR APIs using schemathesis. Main focus will be on the Rodeo Profile, which is a subset of the OGC EDR API. +An experimental validator for OGC EDR APIs. Main focus will be on the Rodeo Profile, which is a subset of the OGC EDR API. ## Who is responsible? @@ -40,17 +40,28 @@ Run manually as noted in [Test it out](#test-it-out), or add it to your CI using ## Overview of architecture +- __init__ includes tests from ogcapi, edrreq and rodeoprofile at startup. Tests are categorized as landing, conformance and collection. +- Landing and conformance tests are run first, in the preflight phase. +- Then schemathesis will validate the OpenAPI spec and run lots of automatic tests, including fuzzing of query parameters. Collection tests are run during this phase. + ## Documentation +- Use --rodeo-profile to force a test against the profile +- Use --strict to also fail on SHOULD requirements. +- Use --log-file debug.log to get all output. For docker variant, see [Test it out](#test-it-out). + ### Limitations - Assuming Openapi 3.1 - Assuming OGC EDR API version 1.2 (draft) - Few, basic tests for now - Will focus more on profiles (limitations within the EDR spec) like than the full EDR spec. -- Use --rodeo-profile to force a test against the profile (will happen automatically if conformance includes the profile) -### Understanding errors +### Testing the sedr code to look for regressions + +For development, source a venv and run `tox p` to run all tests. + +### Understanding errors from schemathesis For each "FAILED" line, you can scroll back to see the full error and, if relevant, with a curl-example to reproduce it. @@ -96,7 +107,7 @@ ERROR sedr/schemat.py - schemathesis.exceptions.SchemaError: Failed to load sche #### Wrong API version / missing conformance link -Sedr supports EDR 1.1, but the API is EDR 1.0. +Sedr wants EDR 1.1, but the API is EDR 1.0. ```python if not requirementA2_2_A5: diff --git a/requirements.txt b/requirements.txt index 0312601..5741aef 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ schemathesis~=3.34 pytest~=8.3 shapely~=2.0 requests~=2.32.0 +rich~=13.8.0 diff --git a/sedr/__init__.py b/sedr/__init__.py index 04609a0..8a4873b 100644 --- a/sedr/__init__.py +++ b/sedr/__init__.py @@ -2,12 +2,16 @@ __author__ = "Lars Falk-Petersen" __license__ = "GPL-3.0" -__version__ = "v0.7.9" +__version__ = "v0.8.0" import sys import pytest + import util import preflight +import edreq12 as edreq +import ogcapi10 as ogcapi +import rodeoprofile10 as rodeoprofile def run_schemat() -> None: @@ -18,6 +22,27 @@ def run_schemat() -> None: def main() -> None: """Run the main program.""" + + # Collect tests to run + util.test_functions["landing"] += edreq.tests_landing + ogcapi.tests_landing + util.test_functions["conformance"] += ( + edreq.tests_conformance + ogcapi.tests_conformance + ) + util.test_functions["collection"] += ( + edreq.tests_collection + ogcapi.tests_collections + ) + if util.args.rodeo_profile: + util.logger.info( + "Including tests for Rodeo profile %s", rodeoprofile.conformance_url + ) + util.test_functions["landing"] += rodeoprofile.tests_landing + util.test_functions["conformance"] += rodeoprofile.tests_conformance + util.test_functions["collection"] += rodeoprofile.tests_collection + + # TODO: include profile tests based on conformance_url, https://github.com/metno/sedr/issues/32 + # if rodeoprofile.conformance_url in conformance_json["conformsTo"]: + # util.args.rodeo_profile = True + if preflight.main(): run_schemat() else: diff --git a/sedr/edreq11.py b/sedr/edreq11.py deleted file mode 100644 index 5afccf6..0000000 --- a/sedr/edreq11.py +++ /dev/null @@ -1,144 +0,0 @@ -"""EDR requirements.""" - -import util - -__edr_version__ = "1.1" -conformance_urls = [ - "http://www.opengis.net/spec/ogcapi-common-1/1.0/conf/core", - "http://www.opengis.net/spec/ogcapi-common-2/1.0/conf/collections", - f"http://www.opengis.net/spec/ogcapi-edr-1/{__edr_version__}/conf/core", -] -openapi_conformance_urls = [ - "http://www.opengis.net/spec/ogcapi-edr-1/1.1/req/oas30", - "http://www.opengis.net/spec/ogcapi-edr-1/1.2/req/oas31", -] - - -def requirementA2_2_A5(jsondata: dict, siteurl="") -> tuple[bool, str]: - """ - OGC API - Environmental Data Retrieval Standard - Version: 1.1 - Requirement Annex A2.2 A5 - - Check if the conformance page contains the required EDR classes. - jsondata should be the "conformsTo"-part of the conformance page. - """ - spec_url = "https://docs.ogc.org/is/19-086r6/19-086r6.html#_c9401fee-54b9-d116-8365-af0f85a8243d" - if "conformsTo" not in jsondata: - return ( - False, - f"Conformance page <{siteurl}conformance> does not contain a conformsTo attribute. See <{spec_url}> for more info.", - ) - for url in conformance_urls: - if url not in jsondata["conformsTo"]: - return ( - False, - f"Conformance page <{siteurl}conformance> does not contain the core edr class {url}. See <{spec_url}> for more info.", - ) - - util.logger.debug( - "requirementA2_2_A5: conformance page contains the required EDR classes." - ) - return True, "" - - -def requirementA2_2_A7(version: int) -> tuple[bool, str]: - """ - OGC API - Environmental Data Retrieval Standard - Version: 1.1 - Requirement Annex A2.2 A7 - - Check if HTTP1.1 was used. - """ - spec_url = "https://docs.ogc.org/is/19-086r6/19-086r6.html#_0d0c25a0-850f-2aa5-9acb-06efcc04d452" - if version == 11: - util.logger.debug("requirementA2_2_A7 HTTP version 1.1 was used.") - return True, "" - - return False, f"HTTP version 1.1 was not used. See <{spec_url}> for more info." - - -def requirementA11_1(jsondata: dict) -> tuple[bool, str]: - """ - OGC API - Environmental Data Retrieval Standard - Version: 1.1 - Requirement A11.1 - - Check if the conformance page contains openapi classes, and that they match our version.""" - spec_url = "https://docs.ogc.org/is/19-086r6/19-086r6.html#_cc7dd5e3-1d54-41ff-b5ba-c5fcb99fa663" - - for url in jsondata["conformsTo"]: - if url in openapi_conformance_urls: - if ( - util.args.openapi_version == "3.1" - and "oas31" in url - or util.args.openapi_version == "3.0" - and "oas30" in url - ): - util.logger.debug("requirementA11_1 Found openapi class <%s>", url) - return True, url - return ( - False, - f"OpenAPI version {util.args.openapi_version} and version in conformance {url} doesn't match. See <{spec_url}> for more info.", - ) - - return ( - False, - f"Conformance page /conformance does not contain an openapi class. See <{spec_url}> for more info.", - ) - - -def requirement9_1(jsondata: dict) -> tuple[bool, str]: - """ - OGC API - Common - Part 1: Core - Version: 1.0.0 - Requirement 9.1 - - Test that the landing page contains required elements. - - TODO: See https://github.com/metno/sedr/issues/6 - """ - spec_ref = "https://docs.ogc.org/is/19-072/19-072.html#_7c772474-7037-41c9-88ca-5c7e95235389" - - if "title" not in jsondata: - return ( - False, - "Landing page does not contain a title. See <{spec_ref}> for more info.", - ) - if "description" not in jsondata: - return ( - False, - "Landing page does not contain a description. See <{spec_ref}> for more info.", - ) - if "links" not in jsondata: - return ( - False, - "Landing page does not contain links. See <{spec_ref}> for more info.", - ) - - service_desc = "" - for link in jsondata["links"]: - if not isinstance(link, dict): - return ( - False, - f"Link {link} is not a dictionary. See <{spec_ref}> for more info.", - ) - if "href" not in link: - return ( - False, - f"Link {link} does not have a href attribute. See <{spec_ref}> for more info.", - ) - if "rel" not in link: - return ( - False, - f"Link {link} does not have a rel attribute. See <{spec_ref}> for more info.", - ) - if link["rel"] == "service-desc": - service_desc = link["href"] - if not service_desc: - return ( - False, - f"Landing page does not contain a service-desc link. See <{spec_ref}> for more info.", - ) - util.logger.debug("requirement9_1 Landing page contains required elements.") - return True, "" diff --git a/sedr/edreq12.py b/sedr/edreq12.py index 9c8dd1a..4b841bf 100644 --- a/sedr/edreq12.py +++ b/sedr/edreq12.py @@ -1,5 +1,6 @@ """EDR requirements.""" +from collections.abc import Callable import util edr_version = "1.2" @@ -16,7 +17,7 @@ ] -def requirementA2_2_A5(jsondata: dict, siteurl="") -> tuple[bool, str]: +def requirementA2_2_A5(jsondata: dict) -> tuple[bool, str]: """ OGC API - Environmental Data Retrieval Standard Version: 1.2 @@ -26,40 +27,21 @@ def requirementA2_2_A5(jsondata: dict, siteurl="") -> tuple[bool, str]: jsondata should be the "conformsTo"-part of the conformance page. """ spec_url = f"{edr_root_url}#req_core_conformance" + if "conformsTo" not in jsondata: - return ( - False, - f"Conformance page <{siteurl}conformance> does not contain a " - f"conformsTo attribute. See <{spec_url}> for more info.", + return False, ( + f"Conformance page does not contain a " + f"conformsTo attribute. See <{spec_url}> for more info." ) + for url in conformance_urls: if url not in jsondata["conformsTo"]: - return ( - False, - f"Conformance page <{siteurl}conformance> does not contain " - f"the core edr class {url}. See <{spec_url}> for more info.", + return False, ( + f"Conformance page does not contain " + f"the core edr class {url}. See <{spec_url}> for more info." ) - util.logger.debug( - "requirementA2_2_A5: conformance page contains the required EDR classes." - ) - return True, "" - - -def requirementA2_2_A7(version: int) -> tuple[bool, str]: - """ - OGC API - Environmental Data Retrieval Standard - Version: 1.2 - Requirement Annex A2.2 A7 - - Check if HTTP1.1 was used. - """ - spec_url = f"{edr_root_url}#_req_core_http" - if version == 11: - util.logger.debug("requirementA2_2_A7 HTTP version 1.1 was used.") - return True, "" - - return False, f"HTTP version 1.1 was not used. See <{spec_url}> for more info." + return True, "Conformance page contains the required EDR classes." def requirementA11_1(jsondata: dict) -> tuple[bool, str]: @@ -77,8 +59,7 @@ def requirementA11_1(jsondata: dict) -> tuple[bool, str]: if ( "oas31" in url or "oas30" in url # TODO: oas30 should be removed ): - util.logger.debug("requirementA11_1 Found openapi class <%s>", url) - return True, url + return True, f"Found openapi class <{url}>. " return ( False, f"OpenAPI version {util.args.openapi_version} and version in " @@ -90,3 +71,31 @@ def requirementA11_1(jsondata: dict) -> tuple[bool, str]: f"Conformance page /conformance does not contain an openapi class. " f"See <{spec_url}> for more info.", ) + + +def requrementA5_2(jsondata: dict) -> tuple[bool, str]: + """ + OGC API - Environmental Data Retrieval Standard + Version: 1.2 + Requirement A5.2 + + Check extent spatial bbox + """ + + spec_url = f"{edr_root_url}#req_core_rc-bbox-definition" + + extent = None + extent = util.parse_spatial_bbox(jsondata) + if extent == [] or len(extent) > 1 or not isinstance(extent, list): + return ( + False, + f"Extent→spatial→bbox should be a list of bboxes with exactly " + f"one bbox in, found {len(extent)} in collection " + f"<{jsondata['id']}>. See {spec_url} for more info.", + ) + return True, f"Extent→spatial→bbox for collection is {extent}" + + +tests_landing: list[Callable[[dict], tuple[bool, str]]] = [] +tests_conformance = [requirementA2_2_A5, requirementA11_1] +tests_collection = [requrementA5_2] diff --git a/sedr/ogcapi10.py b/sedr/ogcapi10.py index 104e3e2..dc9b74a 100644 --- a/sedr/ogcapi10.py +++ b/sedr/ogcapi10.py @@ -1,5 +1,7 @@ """OGC API Common requirements.""" +from collections.abc import Callable +import requests import util ogc_api_common_version = "1.0" @@ -14,7 +16,7 @@ def requirement9_1(jsondata: dict) -> tuple[bool, str]: Test that the landing page contains required elements. - TODO: See https://github.com/metno/sedr/issues/6 + TODO: See https://github.com/metno/sedr/issues/6 - Should landing page in json only be tested if correct conformance class exists? """ spec_ref = f"{ogc_api_common_url}#_7c772474-7037-41c9-88ca-5c7e95235389" @@ -58,5 +60,40 @@ def requirement9_1(jsondata: dict) -> tuple[bool, str]: False, f"Landing page does not contain a service-desc link. See <{spec_ref}> for more info.", ) - util.logger.debug("requirement9_1 Landing page contains required elements.") - return True, "" + return True, "Landing page contains required elements. " + + +def test_conformance_links(jsondata: dict, timeout: int = 10) -> tuple[bool, str]: + """ + OGC API - Common - Part 1: Core + Version: 1.0.0 + + Test that all conformance links are valid and resolves. Not belonging to + any spesific requirement, so not failing unless --strict. + """ + msg = "" + valid = True + for link in jsondata["conformsTo"]: + if link in [ + "http://www.opengis.net/spec/ogcapi-common-2/1.0/conf/conformance", + "http://www.opengis.net/spec/ogcapi-common-2/1.0/conf/collections", + "http://www.opengis.net/spec/ogcapi-edr-1/1.2/req/oas31", + ]: + # TODO: These links are part of the standard but doesn't work, so skipping for now. + msg += f"test_conformance_links Link {link} doesn't resolv, but that is a known issue. " + continue + response = requests.Response() + try: + response = requests.head(url=link, timeout=timeout) + except requests.exceptions.MissingSchema as error: + valid = not util.args.strict + msg += f"test_conformance_links Link <{link}> from /conformance is malformed: {error}). " + if not response.status_code < 400: + valid = not util.args.strict + msg += f"test_conformance_links Link {link} from /conformance is broken (gives status code {response.status_code}). " + return valid, msg + + +tests_landing = [requirement9_1] +tests_conformance = [test_conformance_links] +tests_collections: list[Callable[[dict], tuple[bool, str]]] = [] diff --git a/sedr/preflight.py b/sedr/preflight.py index 276920c..4fa90cb 100644 --- a/sedr/preflight.py +++ b/sedr/preflight.py @@ -3,124 +3,79 @@ import util import requests import json -from urllib.parse import urljoin -import edreq12 as edreq -import ogcapi10 as ogcapi -import rodeoprofile10 as rodeoprofile -def test_site_response(url: str, timeout=10) -> bool: - """Check basic http response.""" - response = requests.get(url, timeout=timeout) - if not response.status_code < 400: - util.logger.error( - "Landing page doesn't respond correctly: status code: %s", - response.status_code, - ) - return False - return True - - -def parse_landing(url, timeout=10) -> tuple[bool, dict]: +def fetch_landing(url: str, timeout: int) -> tuple[bool, dict]: """Test that the landing page contains required elements.""" landing_json = {} - response = requests.get(url, timeout=timeout) + response = requests.Response() try: + response = requests.get(url, timeout=timeout) landing_json = response.json() - except json.decoder.JSONDecodeError: - util.logger.warning("Landing page <%s> is not valid JSON.", url) - return False, landing_json - - landing, requirement9_1_message = ogcapi.requirement9_1(landing_json) - if not landing: - util.logger.error(requirement9_1_message) + except requests.exceptions.ConnectionError as err: + util.logger.error("Unable to get landing page <%s>.\n%s", url, err) return False, landing_json - - requirementA2_2_A7, requirementA2_2_A7_message = edreq.requirementA2_2_A7( - response.raw.version - ) - if not requirementA2_2_A7: - util.logger.error(requirementA2_2_A7_message) + except json.decoder.JSONDecodeError as err: + util.logger.error( + "fetch_landing Landing page <%s> is not valid JSON.\n%s", url, err + ) return False, landing_json - return True, landing_json -def parse_conformance(url: str, timeout: int, landing_json) -> bool: +def fetch_conformance(url: str, timeout: int) -> tuple[bool, dict]: """Test that the conformance page contains required elements.""" + conformance_url = util.build_conformance_url(url) conformance_json = {} - response = requests.get(url, timeout=timeout) + response = requests.Response() try: + response = requests.get(conformance_url, timeout=timeout) conformance_json = response.json() - except json.decoder.JSONDecodeError: - util.logger.warning("Conformance page <%s> is not valid JSON.", url) - return False - - resolves, resolves_message = util.test_conformance_links( - jsondata=conformance_json, timeout=util.args.timeout - ) - if not resolves and util.args.strict: - util.logger.error(resolves_message) - if util.args.strict: - return False - - requirementA2_2_A5, requirementA2_2_A5_message = edreq.requirementA2_2_A5( - jsondata=conformance_json, siteurl=util.args.url - ) - if not requirementA2_2_A5: - util.logger.error(requirementA2_2_A5_message) - if util.args.strict: - return False - - requirementA11_1, requirementA11_1_message = edreq.requirementA11_1( - jsondata=conformance_json - ) - if not requirementA11_1: - util.logger.error(requirementA11_1_message) - if util.args.strict: - return False - - # Rodeo profile - if rodeoprofile.conformance_url in conformance_json["conformsTo"]: - util.args.rodeo_profile = True - if util.args.rodeo_profile: - util.logger.info( - "Including tests for Rodeo profile %s", rodeoprofile.conformance_url - ) - - requirement7_1, requirement7_1_message = rodeoprofile.requirement7_1( - jsondata=conformance_json - ) - if not requirement7_1: - util.logger.error(requirement7_1_message) - if util.args.strict: - return False - - requirement7_2, requirement7_2_message = rodeoprofile.requirement7_2( - jsondata=landing_json, timeout=util.args.timeout + except requests.exceptions.ConnectionError as err: + util.logger.error("Unable to get conformance <%s>.\n%s", url, err) + return False, conformance_json + except json.decoder.JSONDecodeError as err: + util.logger.error( + "Conformance <%s> is not valid JSON:\n%s", conformance_url, err ) - if not requirement7_2: - util.logger.error(requirement7_2_message) - if util.args.strict: - return False - - return True + return False, conformance_json + return True, conformance_json def main(): - conformance_url = urljoin(util.args.url, "/conformance") - - if not test_site_response(util.args.url, util.args.timeout): + # Get landing + landing_is_reachable, landing_json = fetch_landing(util.args.url, util.args.timeout) + if not landing_is_reachable: return False - landing_ok, landing_json = parse_landing(util.args.url, util.args.timeout) - if not landing_ok: + # Run tests for landing + for test_func in util.test_functions["landing"]: + status, msg = test_func(jsondata=landing_json) + if not status: + util.logger.error( + "Test %s failed with message: %s", test_func.__name__, msg + ) + else: + util.logger.info("Test %s passed. (%s)", test_func.__name__, msg) + + # Get conformance + conformance_is_reachable, conformance_json = fetch_conformance( + util.args.url, util.args.timeout + ) + if not conformance_is_reachable: return False - if not parse_conformance(conformance_url, util.args.timeout, landing_json): - return False + # Run tests for conformance page + for test_func in util.test_functions["conformance"]: + status, msg = test_func(conformance_json) + if not status: + util.logger.error( + "Test %s failed with message: %s", test_func.__name__, msg + ) + else: + util.logger.debug("Test %s passed. (%s)", test_func.__name__, msg) util.logger.info("Preflight checks done.") return True diff --git a/sedr/rodeoprofile10.py b/sedr/rodeoprofile10.py index 0988053..a3ee2d3 100644 --- a/sedr/rodeoprofile10.py +++ b/sedr/rodeoprofile10.py @@ -12,8 +12,11 @@ def requirement7_1(jsondata: dict) -> tuple[bool, str]: """ - Check if the conformance page contains the required EDR classes. + RODEO EDR Profile + Version: 0.1.0 + 7.1. Requirements Class "Core" + Check if the conformance page contains the required EDR classes. jsondata should be a valid conformance page json dict. """ spec_url = f"{spec_base_url}#_requirements_class_core" @@ -23,18 +26,18 @@ def requirement7_1(jsondata: dict) -> tuple[bool, str]: f"Conformance page /conformance does not contain the profile " f"class {conformance_url}. See <{spec_url}> for more info.", ) - util.logger.debug("Rodeoprofile Requirement 7.1 OK") - return True, "" + return True, "Conformance contains conformsTo with required EDR classes." -def requirement7_2(jsondata: dict, timeout: int) -> tuple[bool, str]: +def requirement7_2(jsondata: dict, timeout: int = 10) -> tuple[bool, str]: """ RODEO EDR Profile Version: 0.1.0 - 7.2. OpenAPI jsondata should be a valid landing page json dict. + + returns status: bool, message """ spec_url = f"{spec_base_url}#_openapi" openapi_type = "application/vnd.oai.openapi+json;version=3.1" @@ -58,8 +61,8 @@ def requirement7_2(jsondata: dict, timeout: int) -> tuple[bool, str]: if openapi_type not in service_desc_type: return ( False, - f"OpenAPI link service-desc should identify the content as " - f"openAPI and include version. Example <{openapi_type}>. Found: " + "OpenAPI link service-desc should identify the content as openAPI " + f"and include version. Example <{openapi_type}>. Found: " f"<{service_desc_type}>. See <{spec_url}> and <{spec_base_url}" "#_openapi_2> for more info.", ) @@ -110,13 +113,18 @@ def requirement7_2(jsondata: dict, timeout: int) -> tuple[bool, str]: f"Status code: {response.status_code}. See <{spec_url}> for more info.", ) - util.logger.debug("Rodeoprofile Requirement 7.2 OK") - return True, "" + return True, f"{__name__} Landing openapi OK. " def requirement7_3(jsondata: dict) -> tuple[bool, str]: - """Check collection identifier. Can only test B, C. - Should only be tested if --strict is set.""" + """ + RODEO EDR Profile + Version: 0.1.0 + 7.3. Collection identifier + + Check collection identifier. Can only test B, C. + Should only be tested if --strict is set. + """ spec_url = f"{spec_base_url}#_collection_identifier" approved_data_types = [ "insitu-observations", @@ -126,33 +134,38 @@ def requirement7_3(jsondata: dict) -> tuple[bool, str]: "weather_forecast", ] - # B + # B, C try: for t in approved_data_types: if jsondata["id"].startswith(t): break else: return ( - False, + not util.args.strict, f"Collection id SHOULD be from the following list of values: " f"{', '.join(approved_data_types)}. A postfix can be added. " f"Found: <{jsondata['id']}>. See <{spec_url}> for more info.", ) except (json.JSONDecodeError, KeyError) as err: return ( - False, + not util.args.strict, f"Collection must have an id. None found in collection <{jsondata}>." f"Error {err}.", ) - util.logger.debug("Rodeoprofile Requirement 7.3 OK") return ( True, - "", + "Collection id OK. ", ) def requirement7_4(jsondata: dict) -> tuple[bool, str]: - """Check collection title. Can only test A, B.""" + """ + RODEO EDR Profile + Version: 0.1.0 + 7.4. Collection title + + Check collection title. Can only test A, B. + """ spec_url = f"{spec_base_url}#_collection_title" # B @@ -170,18 +183,24 @@ def requirement7_4(jsondata: dict) -> tuple[bool, str]: f"Collection must have a title, but it seems to be missing. See " f"<{spec_url}> and {spec_base_url}#_collection_title_2 for more info.", ) - util.logger.debug("Rodeoprofile Requirement 7.4 OK") return ( True, - "", + "Collection title OK. ", ) def requirement7_5(jsondata: dict) -> tuple[bool, str]: - """Check collection license. Can't test D.""" + """ + RODEO EDR Profile + Version: 0.1.0 + 7.5. Collection license + + Check collection license. Can't test D. + """ spec_url = f"{spec_base_url}#_collection_license" wanted_type = "text/html" wanted_rel = "license" + license_count = 0 # A, B for link in jsondata["links"]: if link["rel"] == wanted_rel: @@ -191,15 +210,26 @@ def requirement7_5(jsondata: dict) -> tuple[bool, str]: f"Collection <{jsondata['id']}> license link should have " f"type='{wanted_type}'. See <{spec_url}> C for more info.", ) - break - else: + license_count += 1 + + if license_count > 1: + return ( + not util.args.strict, + f"Collection <{jsondata['id']}> has more than one license link.", + ) + if license_count < 1: return ( False, f"Collection <{jsondata['id']}> is missing a license link with " f"rel='{wanted_rel}'. See <{spec_url}> A, B for more info.", ) - util.logger.debug("Rodeoprofile Requirement 7.5 OK") + return ( True, - "", + "Collection license OK.", ) + + +tests_landing = [requirement7_2] +tests_conformance = [requirement7_1] +tests_collection = [requirement7_3, requirement7_4, requirement7_5] diff --git a/sedr/schemat.py b/sedr/schemat.py index ec6b697..5459662 100644 --- a/sedr/schemat.py +++ b/sedr/schemat.py @@ -11,7 +11,6 @@ import util import edreq12 as edreq -import rodeoprofile10 as rodeoprofile __author__ = "Lars Falk-Petersen" @@ -96,69 +95,41 @@ def after_call(context, case, response): # noqa: pylint: disable=unused-argumen @settings(max_examples=util.args.iterations, deadline=None) def test_edr_collections(case): """The default testing in function test_api() will fuzz the collections. - This function will test that collections contain EDR spesifics. It will - also require /collections to exist, in accordance with Requirement A.2.2 A.9 - + This function will test that collections contain EDR spesifics. """ global collection_ids, extents # noqa: pylint: disable=global-variable-not-assigned response = case.call() - spec_ref = "https://docs.ogc.org/is/19-086r6/19-086r6.html#_ed0b4d0d-f90a-4a7d-a123-17a1d7849b2d" + spec_ref = f"{edreq.edr_root_url}#_second_tier_collections_tests" assert ( "collections" in json.loads(response.text) ), f"/collections does not contain a collections attribute. See {spec_ref} for more info." - for collection in json.loads(response.text)["collections"]: + for collection_json in json.loads(response.text)["collections"]: # Use url as key for extents. Remove trailing slash from url. - collection_url = collection["links"][0]["href"].rstrip("/") - - collection_ids[collection_url] = collection["id"] - util.logger.debug("test_collections found collection id %s", collection["id"]) - - extent = None - try: - extent = collection["extent"]["spatial"]["bbox"][ - 0 - ] # TODO: assuming only one extent - - # Make sure bbox contains a list of extents, not just an extent - assert isinstance( - extent, list - ), f"Extent→spatial→bbox should be a list of bboxes, not a single bbox. \ - Example [[1, 2, 3, 4], [5, 6, 7, 8]]. Was <{collection['extent']['spatial']['bbox']}>. See {spec_ref} for more info." - extents[collection_url] = tuple(extent) - - util.logger.debug( - "test_collections found extent for %s: %s", collection_url, extent - ) - except AttributeError: - pass - except KeyError as err: - if err.args[0] == "extent": - raise AssertionError( - f"Unable to find extent for collection ID {collection['id']}. Found [{', '.join(collection.keys())}]. See {spec_ref} for more info." - ) from err + collection_url = util.parse_collection_url(collection_json) - if util.args.rodeo_profile: - if util.args.strict: - requirement7_3, requirement7_3_message = rodeoprofile.requirement7_3( - jsondata=collection - ) - if not requirement7_3: - raise AssertionError(requirement7_3_message) + collection_ids[collection_url] = collection_json["id"] + util.logger.debug( + "test_collections found collection id %s", collection_json["id"] + ) - requirement7_4, requirement7_4_message = rodeoprofile.requirement7_4( - jsondata=collection - ) - if not requirement7_4: - raise AssertionError(requirement7_4_message) + # Run edr, ogc, profile tests + for test_func in util.test_functions["collection"]: + status, msg = test_func(jsondata=collection_json) + if not status: + util.logger.error( + "Test %s failed with message: %s", test_func.__name__, msg + ) + raise AssertionError( + f"Test {test_func.__name__} failed with message: {msg}" + ) + util.logger.info("Test %s passed. (%s)", test_func.__name__, msg) - requirement7_5, requirement7_5_message = rodeoprofile.requirement7_5( - jsondata=collection - ) - if not requirement7_5: - raise AssertionError(requirement7_5_message) + # Validation of spatial_bbox done above + extent = util.parse_spatial_bbox(collection_json) + extents[collection_url] = tuple(extent[0]) for p in schema.raw_schema["paths"].keys(): diff --git a/sedr/test_edreq12.py b/sedr/test_edreq12.py new file mode 100644 index 0000000..52e88aa --- /dev/null +++ b/sedr/test_edreq12.py @@ -0,0 +1,35 @@ +"""Unit tests for edreq12.py.""" + +import unittest +import json +import util +import edreq12 as edreq + + +class TestEDR(unittest.TestCase): + __version__ = "testversion" + util.args = util.parse_args(["--url", "https://example.com/"], __version__) + util.logger = util.set_up_logging( + args=util.args, logfile=util.args.log_file, version=__version__ + ) + + def test_requrementA5_2(self): + # Good tests + jsondata = {} + with open("testdata/edrisobaric_collection.json", "r", encoding="utf-8") as f: + jsondata = json.load(f) + ok, _ = edreq.requrementA5_2(jsondata) + self.assertTrue(ok) + + # Bad tests + jsondata = {} + with open( + "testdata/edrisobaric_collection_bad_bbox.json", "r", encoding="utf-8" + ) as f: + jsondata = json.load(f) + ok, _ = edreq.requrementA5_2(jsondata) + self.assertFalse(ok) + + +if __name__ == "__main__": + unittest.main() diff --git a/sedr/test_ogcapi10.py b/sedr/test_ogcapi10.py new file mode 100644 index 0000000..31afa4a --- /dev/null +++ b/sedr/test_ogcapi10.py @@ -0,0 +1,35 @@ +"""Unit tests for ogcapi10.py.""" + +import unittest +import json +import util +import ogcapi10 as ogcapi + + +class TestOGCAPI(unittest.TestCase): + __version__ = "testversion" + util.args = util.parse_args(["--url", "https://example.com/"], __version__) + util.logger = util.set_up_logging( + args=util.args, logfile=util.args.log_file, version=__version__ + ) + + def test_requirement9_1(self): + # Good tests + jsondata = {} + with open("testdata/edrisobaric_landing.json", "r", encoding="utf-8") as f: + jsondata = json.load(f) + ok, _ = ogcapi.requirement9_1(jsondata) + self.assertTrue(ok) + + # Bad tests + jsondata = {} + with open( + "testdata/edrisobaric_landing-bad-desc.json", "r", encoding="utf-8" + ) as f: + jsondata = json.load(f) + ok, _ = ogcapi.requirement9_1(jsondata) + self.assertFalse(ok) + + +if __name__ == "__main__": + unittest.main() diff --git a/sedr/util.py b/sedr/util.py index 26d3e29..b8730d3 100644 --- a/sedr/util.py +++ b/sedr/util.py @@ -4,12 +4,19 @@ import schemathesis import argparse import json -from urllib.parse import urlsplit +from typing import List, Callable, Dict +from urllib.parse import urljoin, urlsplit +from rich.logging import RichHandler __author__ = "Lars Falk-Petersen" __license__ = "GPL-3.0" args = logger = None +test_functions: Dict[str, List[Callable]] = { + "landing": [], + "conformance": [], + "collection": [], +} def parse_args(args, version: str = "") -> argparse.Namespace: @@ -69,10 +76,10 @@ def parse_args(args, version: str = "") -> argparse.Namespace: def set_up_logging(args, logfile=None, version: str = "") -> logging.Logger: """Set up logging.""" - loglevel = logging.DEBUG - logger = logging.getLogger(__file__) logger.setLevel(logging.DEBUG) + FORMAT = "%(asctime)s - %(message)s" + logging.getLogger("requests").setLevel(logging.WARNING) # File if logfile is not None: @@ -91,6 +98,7 @@ def set_up_logging(args, logfile=None, version: str = "") -> logging.Logger: fh = logging.FileHandler(mode="a", filename=logfile) fh.setLevel(logging.DEBUG) + fh.setFormatter(logging.Formatter(FORMAT, datefmt="[%X]")) logger.addHandler(fh) logger.debug( # noqa: pylint: disable=logging-not-lazy f"SEDR version {version} on python {sys.version}, schemathesis " @@ -99,10 +107,9 @@ def set_up_logging(args, logfile=None, version: str = "") -> logging.Logger: ) # Console - stdout_handler = logging.StreamHandler() - stdout_handler.setLevel(loglevel) + stdout_handler = RichHandler(level=logging.INFO) + stdout_handler.setFormatter(logging.Formatter(FORMAT, datefmt="[%X]")) logger.addHandler(stdout_handler) - return logger @@ -134,31 +141,6 @@ def parse_locations(jsondata) -> None: # ) -def test_conformance_links(jsondata: dict, timeout: int) -> tuple[bool, str]: - """Test that all conformance links are valid and resolves.""" - msg = "" - valid = True - for link in jsondata["conformsTo"]: - if link in [ - "http://www.opengis.net/spec/ogcapi-common-2/1.0/conf/conformance", - "http://www.opengis.net/spec/ogcapi-common-2/1.0/conf/collections", - "http://www.opengis.net/spec/ogcapi-edr-1/1.2/req/oas31", - ]: - # TODO: These links are part of the standard but doesn't work, so skipping for now. - msg += f"test_conformance_links Link {link} doesn't resolv, but that is a known issue. " - continue - response = requests.Response() - try: - response = requests.head(url=link, timeout=timeout) - except requests.exceptions.MissingSchema as error: - valid = False - msg += f"test_conformance_links Link <{link}> from /conformance is malformed: {error}). " - if not response.status_code < 400: - valid = False - msg += f"test_conformance_links Link {link} from /conformance is broken (gives status code {response.status_code}). " - return valid, msg - - def locate_openapi_url(url: str, timeout: int) -> str: """Locate the OpenAPI URL based on main URL.""" request = requests.get(url, timeout=timeout) @@ -178,3 +160,19 @@ def locate_openapi_url(url: str, timeout: int) -> str: # Yaml # Xml return "" + + +def build_conformance_url(url: str) -> str: + """Build the conformance URL based on main URL.""" + return urljoin(url, "/conformance") + + +def parse_collection_url(jsondata: dict) -> str: + return jsondata["links"][0]["href"].rstrip("/") + + +def parse_spatial_bbox(jsondata: dict) -> list: + try: + return jsondata["extent"]["spatial"]["bbox"] + except (AttributeError, KeyError) as err: + return [] diff --git a/testdata/edrisobaric_collection.json b/testdata/edrisobaric_collection.json new file mode 100644 index 0000000..fb960b9 --- /dev/null +++ b/testdata/edrisobaric_collection.json @@ -0,0 +1,139 @@ +{ + "id": "weather_forecast", + "title": "IsobaricGRIB - GRIB files", + "description": "These files are used by Avinor ATM systems but possibly also of interest to others. They contain temperature and wind forecasts for a set of isobaric layers (i.e. altitudes having the same pressure). The files are (normally) produced every 6 hours. You can check the time when generated using the Last-Modified header or the `updated` key in `available`. These files are in GRIB2 format (filetype BIN) for the following regions: southern_norway Area 64.25N -1.45W 55.35S 14.51E, resolution .1 degrees? (km?) FIXME It includes every odd-numbered isobaric layer from 1 to 137 (in hundreds of feet?)", + "keywords": [ + "position", + "data", + "api", + "temperature", + "wind", + "forecast", + "isobaric", + "weather_forecast" + ], + "links": [ + { + "href": "https://edrisobaric.k8s.met.no/collections/weather_forecast/", + "rel": "data" + }, + { + "href": "https://data.norge.no/nlod/en/2.0/", + "rel": "license", + "type": "text/html" + }, + { + "href": "https://creativecommons.org/licenses/by/4.0/", + "rel": "license", + "type": "text/html" + } + ], + "extent": { + "spatial": { + "bbox": [ + [ + -1.4499999999999886, + 55.35, + 14.45, + 64.25 + ] + ], + "crs": "GEOGCS[\"Unknown\", DATUM[\"Unknown\", SPHEROID[\"WGS_1984\", 6378137.0, 298.257223563]], PRIMEM[\"Greenwich\",0], UNIT[\"degree\", 0.017453], AXIS[\"Lon\", EAST], AXIS[\"Lat\", NORTH]]" + }, + "temporal": { + "interval": [ + [ + "2024-11-28T18:00:00Z", + "2024-11-29T06:00:00Z" + ] + ], + "values": [ + "2024-11-28T18:00:00+00:00" + ], + "trs": "TIMECRS[\"DateTime\",TDATUM[\"Gregorian Calendar\"],CS[TemporalDateTime,1],AXIS[\"Time (T)\",future]" + }, + "vertical": { + "interval": [ + [ + "850.0" + ], + [ + "100.0" + ] + ], + "values": [ + "850.0", + "750.0", + "700.0", + "600.0", + "500.0", + "450.0", + "400.0", + "350.0", + "300.0", + "275.0", + "250.0", + "225.0", + "200.0", + "150.0", + "100.0" + ], + "vrs": "Vertical Reference System: PressureLevel" + } + }, + "data_queries": { + "position": { + "link": { + "href": "https://edrisobaric.k8s.met.no/collections/weather_forecast/position", + "rel": "data", + "variables": { + "query_type": "position", + "output_formats": [ + "CoverageJSON" + ], + "default_output_format": "CoverageJSON" + } + } + } + }, + "crs": [ + "CRS:84" + ], + "parameter_names": { + "wind_from_direction": { + "type": "Parameter", + "id": "wind_from_direction", + "unit": { + "symbol": { + "value": "˚", + "type": "https://codes.wmo.int/common/unit/_degree_(angle)" + } + }, + "observedProperty": { + "id": "http://vocab.met.no/CFSTDN/en/page/wind_from_direction", + "label": "Wind from direction" + } + }, + "wind_speed": { + "type": "Parameter", + "observedProperty": { + "id": "http://vocab.met.no/CFSTDN/en/page/wind_speed", + "label": "Wind speed" + } + }, + "Air temperature": { + "type": "Parameter", + "id": "Temperature", + "unit": { + "symbol": { + "value": "˚C", + "type": "https://codes.wmo.int/common/unit/_Cel" + } + }, + "observedProperty": { + "id": "http://vocab.met.no/CFSTDN/en/page/air_temperature", + "label": "Air temperature" + } + } + } +} diff --git a/testdata/edrisobaric_collection_bad_bbox.json b/testdata/edrisobaric_collection_bad_bbox.json new file mode 100644 index 0000000..c024e1a --- /dev/null +++ b/testdata/edrisobaric_collection_bad_bbox.json @@ -0,0 +1,137 @@ +{ + "id": "weather_forecast", + "title": "IsobaricGRIB - GRIB files", + "description": "These files are used by Avinor ATM systems but possibly also of interest to others. They contain temperature and wind forecasts for a set of isobaric layers (i.e. altitudes having the same pressure). The files are (normally) produced every 6 hours. You can check the time when generated using the Last-Modified header or the `updated` key in `available`. These files are in GRIB2 format (filetype BIN) for the following regions: southern_norway Area 64.25N -1.45W 55.35S 14.51E, resolution .1 degrees? (km?) FIXME It includes every odd-numbered isobaric layer from 1 to 137 (in hundreds of feet?)", + "keywords": [ + "position", + "data", + "api", + "temperature", + "wind", + "forecast", + "isobaric", + "weather_forecast" + ], + "links": [ + { + "href": "https://edrisobaric.k8s.met.no/collections/weather_forecast/", + "rel": "data" + }, + { + "href": "https://data.norge.no/nlod/en/2.0/", + "rel": "license", + "type": "text/html" + }, + { + "href": "https://creativecommons.org/licenses/by/4.0/", + "rel": "license", + "type": "text/html" + } + ], + "extent": { + "spatial": { + "bbox": [ + -1.4499999999999886, + 55.35, + 14.45, + 64.25 + ], + "crs": "GEOGCS[\"Unknown\", DATUM[\"Unknown\", SPHEROID[\"WGS_1984\", 6378137.0, 298.257223563]], PRIMEM[\"Greenwich\",0], UNIT[\"degree\", 0.017453], AXIS[\"Lon\", EAST], AXIS[\"Lat\", NORTH]]" + }, + "temporal": { + "interval": [ + [ + "2024-11-28T18:00:00Z", + "2024-11-29T06:00:00Z" + ] + ], + "values": [ + "2024-11-28T18:00:00+00:00" + ], + "trs": "TIMECRS[\"DateTime\",TDATUM[\"Gregorian Calendar\"],CS[TemporalDateTime,1],AXIS[\"Time (T)\",future]" + }, + "vertical": { + "interval": [ + [ + "850.0" + ], + [ + "100.0" + ] + ], + "values": [ + "850.0", + "750.0", + "700.0", + "600.0", + "500.0", + "450.0", + "400.0", + "350.0", + "300.0", + "275.0", + "250.0", + "225.0", + "200.0", + "150.0", + "100.0" + ], + "vrs": "Vertical Reference System: PressureLevel" + } + }, + "data_queries": { + "position": { + "link": { + "href": "https://edrisobaric.k8s.met.no/collections/weather_forecast/position", + "rel": "data", + "variables": { + "query_type": "position", + "output_formats": [ + "CoverageJSON" + ], + "default_output_format": "CoverageJSON" + } + } + } + }, + "crs": [ + "CRS:84" + ], + "parameter_names": { + "wind_from_direction": { + "type": "Parameter", + "id": "wind_from_direction", + "unit": { + "symbol": { + "value": "˚", + "type": "https://codes.wmo.int/common/unit/_degree_(angle)" + } + }, + "observedProperty": { + "id": "http://vocab.met.no/CFSTDN/en/page/wind_from_direction", + "label": "Wind from direction" + } + }, + "wind_speed": { + "type": "Parameter", + "observedProperty": { + "id": "http://vocab.met.no/CFSTDN/en/page/wind_speed", + "label": "Wind speed" + } + }, + "Air temperature": { + "type": "Parameter", + "id": "Temperature", + "unit": { + "symbol": { + "value": "˚C", + "type": "https://codes.wmo.int/common/unit/_Cel" + } + }, + "observedProperty": { + "id": "http://vocab.met.no/CFSTDN/en/page/air_temperature", + "label": "Air temperature" + } + } + } +} diff --git a/testdata/edrisobaric_landing-bad-desc.json b/testdata/edrisobaric_landing-bad-desc.json index 77f8f35..66bffaf 100644 --- a/testdata/edrisobaric_landing-bad-desc.json +++ b/testdata/edrisobaric_landing-bad-desc.json @@ -1,6 +1,5 @@ { "title": "EDR isobaric from Grib", - "description": "An EDR API for isobaric data from Grib files", "links": [ { "href": "https://edrisobaric.k8s.met.no/",