Merge pull request #1056 from uw-farlab/pytest_cleanup

Some pytest fixes

compliance_checker/base.py

@@ -192,8 +192,9 @@ def __del__(self):
         inadvertently mutated by other functions.
         """
 
-        cfutil.get_geophysical_variables.cache_clear()
-        cfutil.get_time_variables.cache_clear()
+        if cfutil is not None:
+            cfutil.get_geophysical_variables.cache_clear()
+            cfutil.get_time_variables.cache_clear()
 
 
 class BaseNCCheck:

compliance_checker/protocols/netcdf.py

@@ -91,6 +91,10 @@ def is_remote_netcdf(ds_str):
     else:
         content_type = head_req.headers.get("content-type")
 
+    if content_type is None:
+        return False
+
     # if the Content-Type header returned was "application/x-netcdf",
     # or a netCDF file (not OPeNDAP) we can open this into a Dataset
-    return content_type == "application/x-netcdf"
+    # Add support for application/x-netcdf;ver=4
+    return content_type.split(";")[0] == "application/x-netcdf"

compliance_checker/suite.py

@@ -871,6 +871,11 @@ def load_remote_dataset(self, ds_str):
         content_type = response.headers.get("content-type")
         if content_type.split(";")[0] == "text/xml":
             return self.process_doc(response.content)
+        elif content_type.split(";")[0] == "application/x-netcdf":
+            return Dataset(
+                urlparse(response.url).path,
+                memory=response.content,
+            )
         else:
             raise ValueError(
                 f"Unknown service with content-type: {content_type}",

compliance_checker/tests/test_cf.py

@@ -1794,7 +1794,8 @@ def test_64bit(self):
         dataset = self.load_dataset(STATIC_FILES["ints64"])
         suite = CheckSuite()
         suite.checkers = {"cf": CF1_6Check}
-        suite.run(dataset, "cf")
+        # suite.run(dataset, "cf")
+        suite.run_all(dataset, ["cf"], skip_checks=["cf"])
 
     def test_variable_feature_check(self):
         # non-compliant dataset -- 1/1 fail

compliance_checker/tests/test_cf_integration.py

@@ -245,7 +245,8 @@ def get_results(self, check_results, checksuite):
         ],  # must be specified to load this param at runtime, instead of at collection
     )
     def test_cf_integration(self, loaded_dataset, expected_messages, cs):
-        check_results = cs.run(loaded_dataset, [], "cf")
+        # check_results = cs.run(loaded_dataset, [], "cf")
+        check_results = cs.run_all(loaded_dataset, ["cf"], skip_checks=[])
         scored, out_of, messages = self.get_results(check_results, cs)
 
         assert scored < out_of
@@ -270,14 +271,16 @@ def test_cf_integration(self, loaded_dataset, expected_messages, cs):
         indirect=["loaded_dataset"],
     )
     def test_no_incorrect_errors(self, cs, loaded_dataset, wrong_message):
-        check_results = cs.run(loaded_dataset, [], True, "cf")
+        # check_results = cs.run(loaded_dataset, [], True, "cf")
+        check_results = cs.run_all(loaded_dataset, ["cf"], skip_checks=[])
         messages = self.get_results(check_results, cs)[-1]
 
         assert wrong_message not in "".join(messages)
 
     @pytest.mark.parametrize("loaded_dataset", ["fvcom"], indirect=True)
     def test_fvcom(self, cs, loaded_dataset):
-        check_results = cs.run(loaded_dataset, [], True, "cf")
+        # check_results = cs.run(loaded_dataset, [], True, "cf")
+        check_results = cs.run_all(loaded_dataset, ["cf"], skip_checks=[])
         scored, out_of, messages = self.get_results(check_results, cs)
         assert scored < out_of
 
@@ -305,6 +308,7 @@ def test_ncei_templates(self, cs, loaded_dataset):
         Tests some of the NCEI NetCDF templates, which usually should get a
         perfect score.
         """
-        check_results = cs.run(loaded_dataset, [], "cf")
+        # check_results = cs.run(loaded_dataset, [], "cf")
+        check_results = cs.run_all(loaded_dataset, ["cf"], skip_checks=[])
         scored, out_of, messages = self.get_results(check_results, cs)
         assert scored < out_of

compliance_checker/tests/test_protocols.py

@@ -38,7 +38,10 @@ def test_hyrax():
     """
     Tests that a connection can be made to Hyrax
     """
-    url = "http://test.opendap.org:8080/opendap/ioos/mday_joinExist.ncml"
+    # Returns: error 405
+    # url = "http://test.opendap.org:8080/opendap/ioos/mday_joinExist.ncml"
+    # More direct file
+    url = "http://test.opendap.org:8080/opendap/ioos/mday_joinExist.ncml.dap.nc4"
     cs = CheckSuite()
     ds = cs.load_dataset(url)
     assert ds is not None
@@ -48,13 +51,17 @@ def test_thredds():
     """
     Tests that a connection can be made to a remote THREDDS endpoint
     """
-    url = "http://thredds.ucar.edu/thredds/dodsC/grib/NCEP/GFS/Global_0p25deg_ana/TP"
+    # Returns: error 400
+    # url = "http://thredds.ucar.edu/thredds/dodsC/grib/NCEP/GFS/Global_0p25deg_ana/TP"
+    # Use a smaller dataset
+    url = "https://thredds.ucar.edu/thredds/ncss/grid/grib/NCEP/GFS/Global_0p25deg_ana/TP?var=Temperature_altitude_above_msl&accept=netcdf3"
 
     cs = CheckSuite()
     ds = cs.load_dataset(url)
     assert ds is not None
 
 
+@pytest.mark.skip(reason="The thredds endpoint is no longer serving SOS.")
 def test_sos():
     """
     Tests that a connection can be made to an SOS endpoint

compliance_checker/tests/test_suite.py

@@ -51,16 +51,19 @@ def test_suite(self):
         # BWA: what's the purpose of this test?  Just to see if the suite
         # runs without errors?
         ds = self.cs.load_dataset(static_files["2dim"])
-        self.cs.run(ds, [], "acdd")
+        # self.cs.run(ds, [], "acdd")
+        self.cs.run_all(ds, ["acdd"], skip_checks=[])
 
     def test_suite_pathlib(self):
         path_obj = Path(static_files["2dim"])
         ds = self.cs.load_dataset(path_obj)
-        self.cs.run(ds, [], "acdd")
+        # self.cs.run(ds, [], "acdd")
+        self.cs.run_all(ds, ["acdd"], skip_checks=[])
 
     def test_unicode_formatting(self):
         ds = self.cs.load_dataset(static_files["bad_region"])
-        score_groups = self.cs.run(ds, [], "cf")
+        # score_groups = self.cs.run(ds, [], "cf")
+        score_groups = self.cs.run_all(ds, ["cf"], skip_checks=[])
 
         limit = 2
         for checker, rpair in score_groups.items():
@@ -151,7 +154,8 @@ def test_group_func(self):
         # This is checking for issue #183, where group_func results in
         # IndexError: list index out of range
         ds = self.cs.load_dataset(static_files["bad_data_type"])
-        score_groups = self.cs.run(ds, [], "cf")
+        # score_groups = self.cs.run(ds, [], "cf")
+        score_groups = self.cs.run_all(ds, ["cf"], skip_checks=[])
 
         limit = 2
         for checker, rpair in score_groups.items():
@@ -186,7 +190,8 @@ def test_cdl_file(self):
         # Testing whether you can run compliance checker on a .cdl file
         # Load the cdl file
         ds = self.cs.load_dataset(static_files["test_cdl"])
-        vals = self.cs.run(ds, [], "cf")
+        # vals = self.cs.run(ds, [], "cf")
+        vals = self.cs.run_all(ds, ["cf"], skip_checks=[])
 
         limit = 2
         for checker, rpair in vals.items():
@@ -209,7 +214,8 @@ def test_cdl_file(self):
 
         # Ok now load the nc file that it came from
         ds = self.cs.load_dataset(static_files["test_cdl_nc"])
-        vals = self.cs.run(ds, [], "cf")
+        # vals = self.cs.run(ds, [], "cf")
+        vals = self.cs.run_all(ds, ["cf"], skip_checks=[])
 
         limit = 2
         for checker, rpair in vals.items():
@@ -247,7 +253,8 @@ def test_standard_output_score_header(self):
         of potential issues, rather than the weighted score
         """
         ds = self.cs.load_dataset(static_files["bad_region"])
-        score_groups = self.cs.run(ds, [], "cf")
+        # score_groups = self.cs.run(ds, [], "cf")
+        score_groups = self.cs.run_all(ds, ["cf"], skip_checks=[])
         limit = 2
         groups, errors = score_groups["cf"]
         score_list, all_passed, out_of = self.cs.standard_output(

docs/source/development.md

@@ -0,0 +1,11 @@
+# Developer Notes
+
+## pytest
+
+When running the python test suite, there may be test errors.  Certain
+tests `record` responses to remote queries for information.  If tests
+fail, they will appear to continue to fail as the queries are cached.
+
+To perform tests using fresh queries from remote services, use
+`pytest --disable-vcr`.  In certain cases, clearing the cache is
+also advised, use `pytest --clear-cache`.

docs/source/faq.md

@@ -45,6 +45,8 @@ The Compliance Checker is completely open-source and available on [GitHub](https
 
 ## Disclaimer
 
-The objective of the IOOS Compliance Checker is to check your file against our interpretation of select dataset metadata standards to use as a guideline in generating compliant files.
-The compliance checker should not be considered the authoritative source on whether your file is 100% "compliant".
+The objective of the IOOS Compliance Checker is to check your file against
+our interpretation of select dataset metadata standards to use as a
+guideline in generating compliant files.  The compliance checker should
+not be considered the authoritative source on whether your file is 100% "compliant".
 Instead, we recommend that users use the results as a guide to work towards compliance.

docs/source/index.rst

@@ -10,6 +10,7 @@ Python tool to check your datasets against compliance standards.
    quickintro
    compliance_checker_api
    faq
+   development
 
 Indices and tables
 ==================