Skip to content

Commit

Permalink
linting
Browse files Browse the repository at this point in the history
  • Loading branch information
Jan Griesfeller committed Dec 20, 2024
1 parent 19ed717 commit 89df726
Show file tree
Hide file tree
Showing 8 changed files with 67 additions and 52 deletions.
45 changes: 23 additions & 22 deletions src/pyaro_readers/actrisebas/ActrisEbasReader.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,12 +78,12 @@ class ActrisEbasTestDataNotFoundException(Exception):

class ActrisEbasTimeSeriesReader(AutoFilterReaderEngine.AutoFilterReader):
def __init__(
self,
vars_to_read: list[str] = None,
filters=[],
tqdm_desc: str | None = None,
ts_type: str = "daily",
test_flag: bool = False,
self,
vars_to_read: list[str] = None,
filters=[],
tqdm_desc: str | None = None,
ts_type: str = "daily",
test_flag: bool = False,
):
""" """
self._filename = None
Expand Down Expand Up @@ -226,8 +226,8 @@ def metadata(self):
return self._metadata

def read(
self,
tqdm_desc="reading stations",
self,
tqdm_desc="reading stations",
):
"""
read the data from EBAS thredds server
Expand All @@ -250,12 +250,14 @@ def read(

# put all data variables in the data struct for the moment
for d_idx, _data_var in enumerate(
self._get_ebas_data_vars(
tmp_data,
)
self._get_ebas_data_vars(
tmp_data,
)
):
# look for a standard_name match and return only that variable
std_name = self.get_ebas_data_standard_name(tmp_data, _data_var)
std_name = self.get_ebas_data_standard_name(
tmp_data, _data_var
)
if std_name not in self.standard_names[actris_variable]:
# logger.info(
# f"station {site_name}, file #{f_idx}: skipping variable {_data_var} due to wrong standard name"
Expand Down Expand Up @@ -416,8 +418,8 @@ def _get_ebas_data_vars(self, tmp_data, actris_var: str = None, units: str = Non
try:
# if defined, return only names that match
if (
tmp_data[data_var].attrs["units"]
== self.def_data["actris_std_units"][data_var]
tmp_data[data_var].attrs["units"]
== self.def_data["actris_std_units"][data_var]
):
data_vars.append(data_var)
except KeyError:
Expand All @@ -426,10 +428,10 @@ def _get_ebas_data_vars(self, tmp_data, actris_var: str = None, units: str = Non
return data_vars

def extract_urls(
self,
json_resp: dict,
sites_to_read: list[str] = [],
sites_to_exclude: list[str] = [],
self,
json_resp: dict,
sites_to_read: list[str] = [],
sites_to_exclude: list[str] = [],
) -> dict:
"""
small helper method to extract URLs to download from json reponse from the EBAS API
Expand All @@ -450,11 +452,11 @@ def extract_urls(
# site_data[DISTRIBUTION_ROOT_KEY] is also a list
# search for protocol DISTRIBUTION_PROTOCOL_NAME
for url_idx, distribution_data in enumerate(
site_data[DISTRIBUTION_ROOT_KEY]
site_data[DISTRIBUTION_ROOT_KEY]
):
if (
distribution_data[DISTRIBUTION_PROTOCOL_KEY]
!= DISTRIBUTION_PROTOCOL_NAME
distribution_data[DISTRIBUTION_PROTOCOL_KEY]
!= DISTRIBUTION_PROTOCOL_NAME
):
logger.info(
f"skipping site: {site_name} / proto: {distribution_data[DISTRIBUTION_PROTOCOL_KEY]}"
Expand Down Expand Up @@ -512,4 +514,3 @@ def description(self):

def url(self):
return "https://github.com/metno/pyaro-readers"

Original file line number Diff line number Diff line change
Expand Up @@ -137,7 +137,11 @@ def read(self):
lines = []
_fidx = 0
members = tf.getmembers()
bar = tqdm(desc="extracting tar file...", total=len(members), disable=None)
bar = tqdm(
desc="extracting tar file...",
total=len(members),
disable=None,
)
for _midx, member in enumerate(members):
if fnmatch(member.name, FILE_MASK):
bar.update(1)
Expand Down
21 changes: 10 additions & 11 deletions src/pyaro_readers/aeronetsunreader/AeronetSunTimeseriesReader.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,12 @@

class AeronetSunTimeseriesReader(AutoFilterReaderEngine.AutoFilterReader):
def __init__(
self,
filename,
filters=[],
fill_country_flag: bool = FILL_COUNTRY_FLAG,
tqdm_desc: str | None = None,
ts_type: str = "daily",
self,
filename,
filters=[],
fill_country_flag: bool = FILL_COUNTRY_FLAG,
tqdm_desc: str | None = None,
ts_type: str = "daily",
):
"""open a new Aeronet timeseries-reader
Expand Down Expand Up @@ -97,8 +97,8 @@ def __init__(
self.ts_type = ts_type

def read(
self,
tqdm_desc="reading stations",
self,
tqdm_desc="reading stations",
):
# check if file is a URL
_laststatstr = ""
Expand Down Expand Up @@ -231,7 +231,7 @@ def close(self):
pass

def compute_od_from_angstromexp(
self, to_lambda: float, od_ref: float, lambda_ref: float, angstrom_coeff: float
self, to_lambda: float, od_ref: float, lambda_ref: float, angstrom_coeff: float
) -> float:
"""Compute AOD at specified wavelength
Expand All @@ -258,7 +258,7 @@ def compute_od_from_angstromexp(
return od_ref * (lambda_ref / to_lambda) ** angstrom_coeff

def calc_angstroem_coeff(
self, od1: float, od2: float, wl1: float, wl2: float
self, od1: float, od2: float, wl1: float, wl2: float
) -> float:
"""
small helper method to calculate angstroem coefficient
Expand Down Expand Up @@ -291,4 +291,3 @@ def description(self):

def url(self):
return "https://github.com/metno/pyaro-readers"

2 changes: 1 addition & 1 deletion src/pyaro_readers/ascii2netcdf/Ascii2NetcdfTimeseries.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ def __init__(

self._metadata = self.metadata()

# def read(self):
# def read(self):
self._variables = self._read_file_variables()
station_file = "StationList.csv"
station_filepath = os.path.join(self._directory, station_file)
Expand Down
2 changes: 0 additions & 2 deletions src/pyaro_readers/netcdf_rw/Netcdf_RWTimeseries.py
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,6 @@ def iterate_files(self):
if os.path.exists(file_path):
yield file_path


def metadata(self):
metadata = dict()
date = datetime.datetime.min
Expand Down Expand Up @@ -377,4 +376,3 @@ def url(self):

def read(self):
return self.reader_class().read()

Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ def __init__(
self._tqdm_desc = tqdm_desc

def read(self):
""" read method"""
"""read method"""
# check if the data has been read already
if len(self._data) != 0:
return
Expand All @@ -99,7 +99,9 @@ def read(self):
bar.update(1)
self._process_file(file, self._fill_country_flag)
else:
raise ValueError(f"Given filename {self._filename} is neither a folder or a file")
raise ValueError(
f"Given filename {self._filename} is neither a folder or a file"
)

def _process_file(self, file: Path, fill_country_flag: bool = FILL_COUNTRY_FLAG):
with open(file, newline="") as f:
Expand Down
8 changes: 5 additions & 3 deletions tests/test_ActrisEbasReader.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ def test_flag_list_online(self):
def test_api_reading_small_data_set(self):
engine = pyaro.list_timeseries_engines()[self.engine]
with engine.open(
filters=self.station_filter, vars_to_read=self.vars_to_read
filters=self.station_filter, vars_to_read=self.vars_to_read
) as ts:
self.assertGreaterEqual(len(ts.variables()), 1)
self.assertGreaterEqual(len(ts.stations()), 2)
Expand All @@ -77,8 +77,10 @@ def test_wrappers(self):
engine = pyaro.list_timeseries_engines()[self.engine]
new_var_name = "vmro3"
ebas_var_name = "ozone mass concentration"
with VariableNameChangingReader(engine.open(filters=self.station_filter, vars_to_read=[ebas_var_name]),
reader_to_new={ebas_var_name: new_var_name}) as ts:
with VariableNameChangingReader(
engine.open(filters=self.station_filter, vars_to_read=[ebas_var_name]),
reader_to_new={ebas_var_name: new_var_name},
) as ts:
self.assertEqual(ts.data(new_var_name).variable, new_var_name)
self.assertGreaterEqual(len(ts.variables()), 1)
self.assertGreaterEqual(len(ts.stations()), 2)
Expand Down
29 changes: 19 additions & 10 deletions tests/test_PyerocomBinding.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,15 @@ class TestPyaroReaderPyaerocom(unittest.TestCase):
}
AERONETVAR = "od440aer"
ACTRISEBASVAR = "concso4t"
ACTRISEBASVARLIST = ["concso4t","concso4c"]

ACTRISEBASVARLIST = ["concso4t", "concso4c"]

def test_pyaerocom_aeronet(self):
# test reading via pyaerocom
try:
from pyaerocom.io.pyaro.pyaro_config import PyaroConfig
from pyaerocom.io import ReadUngridded
except ImportError:
assert("pyaerocom not installed")
assert "pyaerocom not installed"
return

data_name = "aeronettest"
Expand All @@ -44,14 +43,20 @@ def test_pyaerocom_actrisebas_single_var(self):
from pyaerocom.io.pyaro.pyaro_config import PyaroConfig
from pyaerocom.io import ReadUngridded
except ImportError:
assert("pyaerocom not installed")
assert "pyaerocom not installed"
return

data_name = "PYARO_actrisebas"
data_id = "actrisebas"
station_filter = {
"stations": {
"include": ["Birkenes II", "Jungfraujoch", "Ispra", "Melpitz", "Westerland"]
"include": [
"Birkenes II",
"Jungfraujoch",
"Ispra",
"Melpitz",
"Westerland",
]
},
}
# needs to be the variable name for actrisebas
Expand All @@ -75,16 +80,22 @@ def test_pyaerocom_actrisebas_many_var(self):
from pyaerocom.io.pyaro.pyaro_config import PyaroConfig
from pyaerocom.io import ReadUngridded
except ImportError:
assert("pyaerocom not installed")
assert "pyaerocom not installed"
return

data_name = "PYARO_actrisebas"
data_id = "actrisebas"
station_filter = {
"stations": {
"include": ["Birkenes II", "Jungfraujoch", "Ispra", "Melpitz", "Westerland"]
"include": [
"Birkenes II",
"Jungfraujoch",
"Ispra",
"Melpitz",
"Westerland",
]
},
"variables": {"include": ["concso4t", "concso4c"]}
"variables": {"include": ["concso4t", "concso4c"]},
}
# needs to be the variable name for actrisebas, but PyaroConfig wants this to a string and not a list
# (the pydantic setup is too pedantic)
Expand All @@ -104,7 +115,5 @@ def test_pyaerocom_actrisebas_many_var(self):
self.assertIn(url[1], data.contains_vars)




if __name__ == "__main__":
unittest.main()

0 comments on commit 89df726

Please sign in to comment.