Update metadata

metno · Nov 6, 2024 · cc6ae80 · cc6ae80
1 parent 100ffc5
commit cc6ae80
Show file tree

Hide file tree

Showing 3 changed files with 52 additions and 76,649 deletions.
diff --git a/src/pyaro_readers/eeareader/EEATimeseriesReader.py b/src/pyaro_readers/eeareader/EEATimeseriesReader.py
@@ -188,7 +188,13 @@ def __init__(
     ):
         data_directory = Path(filename_or_obj_or_url)
         metadata_file = data_directory.joinpath("metadata.csv")
-        self._metadata = polars.read_csv(metadata_file)
+        self._metadata = polars.read_csv(
+            metadata_file,
+            schema_overrides={
+                "Air Quality Station Nat Code": str,
+                "Detection Limit": float,
+            },
+        )
 
         # Vocabulary as found at https://dd.eionet.europa.eu/vocabulary/aq/pollutant
         pollutant_file_bytes = importlib.resources.files(

diff --git a/tests/test_EEAReader.py b/tests/test_EEAReader.py
@@ -4,6 +4,9 @@
 
 import pyaro
 import pyaro.timeseries
+import numpy as np
+
+from pyaro_readers.eeareader import EEATimeseriesReader
 
 
 class TestEEATimeSeriesReader(unittest.TestCase):
@@ -24,49 +27,42 @@ class TestEEATimeSeriesReader(unittest.TestCase):
     def test_0engine(self):
         self.assertIn(self.engine, pyaro.list_timeseries_engines())
 
-    # def test_1open_files(self):
-    #     with pyaro.open_timeseries(
-    #         self.engine,
-    #         self.testdata_dir,
-    #         filters={"variables": {"include": ["PM10", "SO2"]}},
-    #     ) as ts:
-    #         self.assertGreaterEqual(len(ts.variables()), 2)
-    #         self.assertGreaterEqual(len(ts.stations()), 2)
-    #         for var in ts.variables():
-    #             assert var in self.test_vars
-
-
-def test_eea_reader2():
-    # Only if PPI is available
-    from pyaro_readers.eeareader import EEATimeseriesReader
-    import pyaro.timeseries
-
-    filters = pyaro.timeseries.FilterCollection(
-        {
-            "time_bounds": {
-                "start_include": [("2019-01-01 00:00:00", "2019-12-24 00:00:00")]
-            },
-            "stations": {
-                "exclude": ["GB/GB_SamplingPoint_61718", "GB/GB_SamplingPoint_99"]
-            },
-            "countries": {"include": ["UK"]},
-        }
-    )
-
-    reader = EEATimeseriesReader(
-        "/lustre/storeB/project/aerocom/aerocom1/AEROCOM_OBSDATA/EEA-AQDS/download",
-        filters=filters,
-        enable_progressbar=True,
-    )
-
-    _ = reader.stations()
-    eea_variables = reader.variables()
-    known_variables = {"PM2.5", "PM10", "NO2"}
-    assert known_variables.issubset(eea_variables)
-
-    data = reader.data("PM2.5")
-    _ = data.altitudes
-    _ = data.values
+    def test_1open_files(self):
+        with pyaro.open_timeseries(
+            self.engine,
+            self.testdata_dir,
+        ) as ts:
+            self.assertGreaterEqual(len(ts.variables()), 2)
+            self.assertGreaterEqual(len(ts.stations()), 2)
+            self.assertTrue(set(self.test_vars).issubset(ts.variables()))
+
+    def test_eea_reader(self):
+        filters = pyaro.timeseries.FilterCollection(
+            {
+                "time_bounds": {
+                    "start_include": [("2019-01-01 00:00:00", "2023-12-24 00:00:00")]
+                },
+                "stations": {
+                    "exclude": ["NO/SPO_NO0151A_8_4768", "NO/SPO_NO0111A_9_1691"]
+                },
+                "countries": {"include": ["NO", "LU"]},
+            }
+        )
+
+        # If PPI is available one can use the following:
+        # "/lustre/storeB/project/aerocom/aerocom1/AEROCOM_OBSDATA/EEA-AQDS/download",
+        reader = EEATimeseriesReader(
+            self.testdata_dir,
+            filters=filters,
+        )
+
+        _ = reader.stations()
+
+        for species in self.test_vars:
+            data = reader.data(species)
+            _ = data.values
+            alts = data.altitudes
+            self.assertFalse(np.any(np.isnan(alts)))
 
 
 if __name__ == "__main__":