diff --git a/disdrodb/api/create_directories.py b/disdrodb/api/create_directories.py index 1afb8a1f..3d4133e9 100644 --- a/disdrodb/api/create_directories.py +++ b/disdrodb/api/create_directories.py @@ -415,7 +415,6 @@ def create_initial_station_structure( data_source=data_source, campaign_name=campaign_name, station_name=station_name, - product="RAW", ) # Add default station issue file create_station_issue( @@ -440,7 +439,7 @@ def create_test_archive(test_base_dir, data_source, campaign_name, station_name, This enable to then test data download and DISDRODB processing. """ # Check test_base_dir is not equal to true base_dir - if test_base_dir == get_base_dir(): + if test_base_dir == get_base_dir(base_dir): raise ValueError("Provide a test_base_dir directory different from the true DISDRODB base directory !") # Create test DISDRODB base directory diff --git a/disdrodb/data_transfer/scripts/disdrodb_upload_station.py b/disdrodb/data_transfer/scripts/disdrodb_upload_station.py index 878c7884..4d674984 100644 --- a/disdrodb/data_transfer/scripts/disdrodb_upload_station.py +++ b/disdrodb/data_transfer/scripts/disdrodb_upload_station.py @@ -22,8 +22,8 @@ import click -from disdrodb.data_transfer.upload_data import click_base_dir_option, click_upload_options -from disdrodb.utils.scripts import click_station_arguments +from disdrodb.data_transfer.upload_data import click_upload_options +from disdrodb.utils.scripts import click_base_dir_option, click_station_arguments sys.tracebacklimit = 0 # avoid full traceback error if occur diff --git a/disdrodb/data_transfer/upload_data.py b/disdrodb/data_transfer/upload_data.py index f3e63328..a91368fc 100644 --- a/disdrodb/data_transfer/upload_data.py +++ b/disdrodb/data_transfer/upload_data.py @@ -111,6 +111,13 @@ def _filter_already_uploaded(metadata_filepaths: List[str], force: bool) -> List return filtered +def _check_valid_platform(platform): + """Check upload platform validity.""" + valid_platform = ["zenodo", "sandbox.zenodo"] + if platform not in valid_platform: + raise NotImplementedError(f"Invalid platform {platform}. Valid platforms are {valid_platform}.") + + def upload_station( data_source: str, campaign_name: str, @@ -145,6 +152,8 @@ def upload_station( The default is force=False. """ + _check_valid_platform(platform) + # Define metadata_filepath metadata_filepath = define_metadata_filepath( data_source=data_source, @@ -162,11 +171,8 @@ def upload_station( if platform == "zenodo": upload_station_to_zenodo(metadata_filepath, sandbox=False) - elif platform == "sandbox.zenodo": # Only for testing purposes, not available through CLI + else: # platform == "sandbox.zenodo": # Only for testing purposes, not available through CLI upload_station_to_zenodo(metadata_filepath, sandbox=True) - else: - valid_platform = ["zenodo", "sandbox.zenodo"] - raise NotImplementedError(f"Invalid platform {platform}. Valid platforms are {valid_platform}.") def upload_archive( @@ -206,6 +212,8 @@ def upload_archive( If not provided (None), all stations will be uploaded. The default is station_name=None. """ + _check_valid_platform(platform) + # Get list metadata metadata_filepaths = get_list_metadata( **kwargs, diff --git a/disdrodb/data_transfer/zenodo.py b/disdrodb/data_transfer/zenodo.py index eb5bc5b5..da206000 100644 --- a/disdrodb/data_transfer/zenodo.py +++ b/disdrodb/data_transfer/zenodo.py @@ -151,11 +151,20 @@ def _define_creators_list(metadata): list_names = re.split(";|,", metadata["authors"]) list_identifier = re.split(";|,", metadata["authors_url"]) list_affiliation = re.split(";|,", metadata["institution"]) + + # Check identifier fields match the number of specified authors + # - If not, set identifier to "" if len(list_names) != len(list_identifier): - # print("Impossible to automatically assign identifier to authors. Length mismatch.") list_identifier = [""] * len(list_names) + + # If affiliation is only one --> Assume is the same for everyone if len(list_affiliation) == 1: list_affiliation = list_affiliation * len(list_names) + # If more than one affiliation, but does not match list of authors, set to "" + if len(list_affiliation) > 1 and len(list_affiliation) != len(list_names): + list_affiliation = [""] * len(list_names) + + # Create info dictionary of each author list_creator_dict = [] for name, identifier, affiliation in zip(list_names, list_identifier, list_affiliation): creator_dict = {} @@ -224,7 +233,7 @@ def upload_station_to_zenodo(metadata_filepath: str, sandbox: bool = True) -> st os.remove(station_zip_filepath) except Exception as e: os.remove(station_zip_filepath) - raise ValueError(f"The upload on Zenodo has failed: {e}.") + raise ValueError(f"{station_zip_filepath} The upload on Zenodo has failed: {e}.") # Add the disdrodb_data_url information to the metadata print(" - The station metadata 'disdrodb_data_url' key has been updated with the remote url") diff --git a/disdrodb/issue/writer.py b/disdrodb/issue/writer.py index 5d9f24c8..48ef970d 100644 --- a/disdrodb/issue/writer.py +++ b/disdrodb/issue/writer.py @@ -50,7 +50,7 @@ def _write_issue_docs(f): return None -def _write_issue(filepath: str, timesteps: list = None, time_periods: list = None) -> None: +def write_issue(filepath: str, timesteps: list = None, time_periods: list = None) -> None: """Write the issue YAML file. Parameters @@ -92,35 +92,6 @@ def _write_issue(filepath: str, timesteps: list = None, time_periods: list = Non return None -def write_issue_dict(filepath: str, issue_dict: dict) -> None: - """Write the issue YAML file. - - Parameters - ---------- - filepath : str - Filepath of the issue YAML to write. - issue_dict : dict - Issue dictionary - """ - _write_issue( - filepath=filepath, - timesteps=issue_dict.get("timesteps", None), - time_periods=issue_dict.get("time_periods", None), - ) - - -def write_default_issue(filepath: str) -> None: - """Write an empty issue YAML file. - - Parameters - ---------- - filepath : str - Filepath of the issue YAML to write. - """ - _write_issue(filepath=filepath) - return None - - def create_station_issue(data_source, campaign_name, station_name, base_dir=None): """Write an empty YAML issue YAML file for a DISDRODB station. @@ -155,6 +126,6 @@ def create_station_issue(data_source, campaign_name, station_name, base_dir=None issue_dir = os.path.dirname(issue_filepath) os.makedirs(issue_dir, exist_ok=True) # Write issue file - write_default_issue(filepath=issue_filepath) + write_issue(filepath=issue_filepath) print(f"An empty issue YAML file for station {station_name} has been created .") return None diff --git a/disdrodb/l0/l0_processing.py b/disdrodb/l0/l0_processing.py index 68d3a5fb..d746758b 100644 --- a/disdrodb/l0/l0_processing.py +++ b/disdrodb/l0/l0_processing.py @@ -22,6 +22,7 @@ import functools import logging import os +import shutil import time import dask @@ -39,6 +40,7 @@ define_l0a_filepath, define_l0b_filepath, define_l0b_station_dir, + define_station_dir, get_disdrodb_path, ) from disdrodb.configs import get_base_dir @@ -785,7 +787,7 @@ def run_l0b_from_nc( return None -def run_l0b_concat(processed_dir, station_name, remove=False, verbose=False): +def run_l0b_concat(processed_dir, station_name, verbose=False): """Concatenate all L0B netCDF files into a single netCDF file. The single netCDF file is saved at /L0B. @@ -836,13 +838,6 @@ def run_l0b_concat(processed_dir, station_name, remove=False, verbose=False): ds.close() del ds - # -------------------------------------------------------------------------. - # If remove = True, remove all the single files - if remove: - log_info(logger=logger, msg="Removal of single L0B files started.", verbose=verbose) - _ = [os.remove(filepath) for filepath in filepaths] - log_info(logger=logger, msg="Removal of single L0B files ended.", verbose=verbose) - # -------------------------------------------------------------------------. # Close the file logger close_logger(logger) @@ -948,6 +943,7 @@ def run_l0b_station( verbose: bool = True, parallel: bool = True, debugging_mode: bool = False, + remove_l0a: bool = False, base_dir: str = None, ): """ @@ -1005,6 +1001,18 @@ def run_l0b_station( parallel=parallel, ) + if remove_l0a: + station_dir = define_station_dir( + base_dir=base_dir, + product="L0A", + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + log_info(logger=logger, msg="Removal of single L0A files started.", verbose=verbose) + shutil.rmtree(station_dir) + log_info(logger=logger, msg="Removal of single L0A files ended.", verbose=verbose) + def run_l0b_concat_station( # Station arguments @@ -1053,9 +1061,20 @@ def run_l0b_concat_station( run_l0b_concat( processed_dir=processed_dir, station_name=station_name, - remove=remove_l0b, verbose=verbose, ) + if remove_l0b: + station_dir = define_station_dir( + base_dir=base_dir, + product="L0B", + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + log_info(logger=logger, msg="Removal of single L0B files started.", verbose=verbose) + shutil.rmtree(station_dir) + log_info(logger=logger, msg="Removal of single L0B files ended.", verbose=verbose) + ####---------------------------------------------------------------------------. diff --git a/disdrodb/l0/routines.py b/disdrodb/l0/routines.py index fa35bcea..1c6d75de 100644 --- a/disdrodb/l0/routines.py +++ b/disdrodb/l0/routines.py @@ -20,8 +20,6 @@ import datetime import logging -import os -import shutil import time import click @@ -107,6 +105,17 @@ def click_l0_processing_options(function: object): return function +def click_remove_l0a_option(function: object): + function = click.option( + "--remove_l0a", + type=bool, + show_default=True, + default=False, + help="If true, remove the L0A files once the L0B processing is terminated.", + )(function) + return function + + def click_l0_archive_options(function: object): """Click command line arguments for L0 processing archiving of a station. @@ -228,6 +237,7 @@ def run_disdrodb_l0b_station( debugging_mode: bool = False, parallel: bool = True, base_dir: str = None, + remove_l0a: bool = False, ): """Run the L0B processing of a station calling disdrodb_run_l0b_station in the terminal.""" # Define command @@ -247,6 +257,8 @@ def run_disdrodb_l0b_station( str(debugging_mode), "--parallel", str(parallel), + "--remove_l0a", + str(remove_l0a), "--base_dir", str(base_dir), ] @@ -358,7 +370,6 @@ def run_disdrodb_l0_station( Base directory of DISDRODB. Format: <...>/DISDRODB If None (the default), the disdrodb config variable 'dir' is used. """ - from disdrodb.api.path import get_disdrodb_path # ---------------------------------------------------------------------. t_i = time.time() @@ -394,20 +405,9 @@ def run_disdrodb_l0_station( verbose=verbose, debugging_mode=debugging_mode, parallel=parallel, + remove_l0a=remove_l0a, ) - # ------------------------------------------------------------------------. - # Remove L0A station directory if remove_l0a = True and l0b_processing = True - if l0b_processing and remove_l0a: - campaign_dir = get_disdrodb_path( - base_dir=base_dir, - product="L0A", - data_source=data_source, - campaign_name=campaign_name, - ) - station_product_dir = os.path.join(campaign_dir, "L0A", station_name) - shutil.rmtree(station_product_dir) - # ------------------------------------------------------------------------. # If l0b_concat=True, concat the netCDF in a single file if l0b_concat: @@ -656,6 +656,7 @@ def run_disdrodb_l0b( debugging_mode: bool = False, parallel: bool = True, base_dir: str = None, + remove_l0a: bool = False, ): run_disdrodb_l0( base_dir=base_dir, @@ -666,7 +667,7 @@ def run_disdrodb_l0b( l0a_processing=False, l0b_processing=True, l0b_concat=False, - remove_l0a=False, + remove_l0a=remove_l0a, remove_l0b=False, # Processing options force=force, @@ -676,6 +677,7 @@ def run_disdrodb_l0b( ) +####---------------------------------------------------------------------------. def run_disdrodb_l0b_concat( data_sources=None, campaign_names=None, diff --git a/disdrodb/l0/scripts/disdrodb_run_l0b.py b/disdrodb/l0/scripts/disdrodb_run_l0b.py index 518fa5bd..f56b3c31 100644 --- a/disdrodb/l0/scripts/disdrodb_run_l0b.py +++ b/disdrodb/l0/scripts/disdrodb_run_l0b.py @@ -22,6 +22,7 @@ from disdrodb.l0.routines import ( click_l0_processing_options, click_l0_stations_options, + click_remove_l0a_option, ) from disdrodb.utils.scripts import click_base_dir_option, parse_arg_to_list @@ -31,6 +32,7 @@ @click.command() @click_l0_stations_options @click_l0_processing_options +@click_remove_l0a_option @click_base_dir_option def disdrodb_run_l0b( # L0 disdrodb stations options @@ -42,6 +44,7 @@ def disdrodb_run_l0b( verbose: bool = True, parallel: bool = True, debugging_mode: bool = False, + remove_l0a: bool = False, base_dir: str = None, ): """ @@ -107,6 +110,7 @@ def disdrodb_run_l0b( verbose=verbose, debugging_mode=debugging_mode, parallel=parallel, + remove_l0a=remove_l0a, ) return None diff --git a/disdrodb/l0/scripts/disdrodb_run_l0b_station.py b/disdrodb/l0/scripts/disdrodb_run_l0b_station.py index a2fb0eb0..d1ab5c3a 100644 --- a/disdrodb/l0/scripts/disdrodb_run_l0b_station.py +++ b/disdrodb/l0/scripts/disdrodb_run_l0b_station.py @@ -18,7 +18,7 @@ import click -from disdrodb.l0.routines import click_l0_processing_options +from disdrodb.l0.routines import click_l0_processing_options, click_remove_l0a_option from disdrodb.utils.scripts import click_base_dir_option, click_station_arguments sys.tracebacklimit = 0 # avoid full traceback error if occur @@ -30,6 +30,7 @@ @click.command() @click_station_arguments @click_l0_processing_options +@click_remove_l0a_option @click_base_dir_option def disdrodb_run_l0b_station( # Station arguments @@ -41,6 +42,7 @@ def disdrodb_run_l0b_station( verbose: bool = True, parallel: bool = True, debugging_mode: bool = False, + remove_l0a: bool = False, base_dir: str = None, ): """Run the L0B processing of a specific DISDRODB station from the terminal. @@ -115,6 +117,7 @@ def disdrodb_run_l0b_station( verbose=verbose, debugging_mode=debugging_mode, parallel=parallel, + remove_l0a=remove_l0a, base_dir=base_dir, ) diff --git a/disdrodb/metadata/writer.py b/disdrodb/metadata/writer.py index 52228496..69fe4746 100644 --- a/disdrodb/metadata/writer.py +++ b/disdrodb/metadata/writer.py @@ -20,7 +20,6 @@ import os -from disdrodb.api.info import infer_campaign_name_from_path, infer_data_source_from_path from disdrodb.api.path import define_metadata_filepath from disdrodb.metadata.manipulation import sort_metadata_dictionary from disdrodb.metadata.standards import get_valid_metadata_keys @@ -48,35 +47,6 @@ def get_default_metadata_dict() -> dict: return attrs -def write_default_metadata(filepath: str) -> None: - """Write a default YAML metadata file at the specified filepath. - - Parameters - ---------- - filepath : str - File path - """ - # Get default metadata dict - metadata = get_default_metadata_dict() - - # Try infer the data_source, campaign_name and station_name from filepath - try: - campaign_name = infer_campaign_name_from_path(filepath) - data_source = infer_data_source_from_path(filepath) - station_name = os.path.basename(filepath).split(".yml")[0] - metadata["data_source"] = data_source - metadata["campaign_name"] = campaign_name - metadata["station_name"] = station_name - except Exception: - pass - - # Write the metadata - metadata = sort_metadata_dictionary(metadata) - - write_yaml(metadata, filepath=filepath, sort_keys=False) - return None - - def create_station_metadata(data_source, campaign_name, station_name, base_dir=None, product="RAW"): """Write a default (semi-empty) YAML metadata file for a DISDRODB station. @@ -111,10 +81,22 @@ def create_station_metadata(data_source, campaign_name, station_name, base_dir=N ) if os.path.exists(metadata_filepath): raise ValueError("A metadata YAML file already exists at {metadata_filepath}.") + # Create metadata dir if not existing metadata_dir = os.path.dirname(metadata_filepath) os.makedirs(metadata_dir, exist_ok=True) - # Write metadata file - write_default_metadata(filepath=metadata_filepath) - print(f"An empty metadata YAML file for station {station_name} has been created .") + + # Get default metadata dict + metadata = get_default_metadata_dict() + + # Try infer the data_source, campaign_name and station_name from filepath + metadata["data_source"] = data_source + metadata["campaign_name"] = campaign_name + metadata["station_name"] = station_name + + # Write the metadata + metadata = sort_metadata_dictionary(metadata) + write_yaml(metadata, filepath=metadata_filepath, sort_keys=False) + + print(f"An empty default metadata YAML file for station {station_name} has been created .") return None diff --git a/disdrodb/tests/conftest.py b/disdrodb/tests/conftest.py index f17ca1ca..c0ca4151 100644 --- a/disdrodb/tests/conftest.py +++ b/disdrodb/tests/conftest.py @@ -85,7 +85,7 @@ def create_fake_issue_file( station_name="station_name", ): from disdrodb.api.path import define_issue_filepath - from disdrodb.issue.writer import write_issue_dict + from disdrodb.issue.writer import write_issue # Define issue filepath issue_filepath = define_issue_filepath( @@ -99,8 +99,11 @@ def create_fake_issue_file( os.makedirs(os.path.dirname(issue_filepath), exist_ok=True) # Write issue - write_issue_dict(issue_filepath, issue_dict=issue_dict) - + write_issue( + filepath=issue_filepath, + timesteps=issue_dict.get("timesteps", None), + time_periods=issue_dict.get("time_periods", None), + ) # Return filepath return str(issue_filepath) diff --git a/disdrodb/tests/test_api/test_create_directories.py b/disdrodb/tests/test_api/test_create_directories.py index 0d3d70bf..39c6e6b3 100644 --- a/disdrodb/tests/test_api/test_create_directories.py +++ b/disdrodb/tests/test_api/test_create_directories.py @@ -330,6 +330,16 @@ def test_create_test_archive(tmp_path): campaign_name = "CAMPAIGN_NAME" data_source = "DATA_SOURCE" station_name = "station_name" + + # Check that if raise error if providing the base_dir + with pytest.raises(ValueError): + create_test_archive( + test_base_dir=base_dir, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + # Create initial station structure create_initial_station_structure( base_dir=base_dir, @@ -362,15 +372,6 @@ def test_create_test_archive(tmp_path): assert os.path.exists(metadata_filepath) assert os.path.exists(issue_filepath) - # Check that if called once again, it raise error - with pytest.raises(ValueError): - create_initial_station_structure( - base_dir=base_dir, - data_source=data_source, - campaign_name=campaign_name, - station_name=station_name, - ) - def test_check_campaign_name_consistency(tmp_path): base_dir = tmp_path / "DISDRODB" diff --git a/disdrodb/tests/test_data_transfer/test_download_data.py b/disdrodb/tests/test_data_transfer/test_download_data.py index 8a2e3171..4d846b8f 100644 --- a/disdrodb/tests/test_data_transfer/test_download_data.py +++ b/disdrodb/tests/test_data_transfer/test_download_data.py @@ -16,7 +16,7 @@ # # You should have received a copy of the GNU General Public License # # along with this program. If not, see . # # -----------------------------------------------------------------------------. -# """Test DISDRODB download utility.""" +"""Test DISDRODB download utility.""" import os @@ -103,6 +103,7 @@ def test_download_without_any_remote_url(tmp_path, requests_mock, mocker, disdro station_name=station_name, force=force, ) + # Check download archive run download_archive( base_dir=str(base_dir), @@ -113,6 +114,34 @@ def test_download_without_any_remote_url(tmp_path, requests_mock, mocker, disdro ) +def test_download_station_only_with_valid_metadata(tmp_path): + """Test download of archive stations is not stopped by single stations download errors.""" + base_dir = tmp_path / "DISDRODB" + data_source = "test_data_source" + campaign_name = "test_campaign_name" + station_name = "test_station_name" + + metadata_dict = {} + metadata_dict["station_name"] = "ANOTHER_STATION_NAME" + metadata_dict["disdrodb_data_url"] = TEST_ZIP_FPATH + + _ = create_fake_metadata_file( + base_dir=base_dir, + metadata_dict=metadata_dict, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + + with pytest.raises(ValueError): + download_station( + base_dir=str(base_dir), + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + + @pytest.mark.parametrize("force", [True, False]) def test_download_station(tmp_path, force): """Test download station data.""" @@ -188,7 +217,7 @@ def test_download_archive(tmp_path, force, existing_data): base_dir=base_dir, data_source=data_source, campaign_name=campaign_name, station_name=station_name ) - # Check download_station raise error if existing data and force=False + # Check download_archive does not raise error if existing data and force=False download_archive( base_dir=str(base_dir), data_sources=data_source, diff --git a/disdrodb/tests/test_data_transfer/test_scripts.py b/disdrodb/tests/test_data_transfer/test_scripts.py new file mode 100644 index 00000000..abe072af --- /dev/null +++ b/disdrodb/tests/test_data_transfer/test_scripts.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 + +# -----------------------------------------------------------------------------. +# Copyright (c) 2021-2023 DISDRODB developers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# -----------------------------------------------------------------------------. +"""Test DISDRODB Download/Upload commands.""" + +from click.testing import CliRunner + +from disdrodb.data_transfer.scripts.disdrodb_upload_station import disdrodb_upload_station +from disdrodb.data_transfer.scripts.disdrodb_upload_archive import disdrodb_upload_archive +from disdrodb.data_transfer.scripts.disdrodb_download_station import disdrodb_download_station +from disdrodb.data_transfer.scripts.disdrodb_download_archive import disdrodb_download_archive +from disdrodb.tests.conftest import create_fake_metadata_file + + +TEST_ZIP_FPATH = "https://raw.githubusercontent.com/ltelab/disdrodb/main/disdrodb/tests/data/test_data_download/station_files.zip" # noqa + + +def test_disdrodb_upload_station(tmp_path): + """Test the disdrodb_upload_station command.""" + base_dir = tmp_path / "DISDRODB" + data_source = "DATA_SOURCE" + campaign_name = "CAMPAIGN_NAME" + station_name = "station_name" + + # - Add fake metadata + _ = create_fake_metadata_file( + base_dir=base_dir, + product="RAW", + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + + runner = CliRunner() + runner.invoke( + disdrodb_upload_station, + [data_source, campaign_name, station_name, + "--base_dir", base_dir], + ) + + +def test_disdrodb_upload_archive(tmp_path): + """Test the disdrodb_upload_archive command.""" + base_dir = tmp_path / "DISDRODB" + data_source = "DATA_SOURCE" + campaign_name = "CAMPAIGN_NAME" + station_name = "station_name" + + # - Add fake metadata + _ = create_fake_metadata_file( + base_dir=base_dir, + product="RAW", + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + + runner = CliRunner() + runner.invoke( + disdrodb_upload_archive, + ["--base_dir", base_dir], + ) + + +def test_disdrodb_download_station(tmp_path): + """Test the disdrodb_download_station command.""" + base_dir = tmp_path / "DISDRODB" + data_source = "DATA_SOURCE" + campaign_name = "CAMPAIGN_NAME" + station_name = "station_name" + + metadata_dict = {} + metadata_dict["disdrodb_data_url"] = TEST_ZIP_FPATH + + # - Add fake metadata + _ = create_fake_metadata_file( + base_dir=base_dir, + product="RAW", + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + metadata_dict=metadata_dict, + ) + + runner = CliRunner() + runner.invoke( + disdrodb_download_station, + [data_source, campaign_name, station_name, + "--base_dir", base_dir], + ) + + +def test_disdrodb_download_archive(tmp_path): + """Test the disdrodb_download_archive command.""" + base_dir = tmp_path / "DISDRODB" + data_source = "DATA_SOURCE" + campaign_name = "CAMPAIGN_NAME" + station_name = "station_name" + + metadata_dict = {} + metadata_dict["disdrodb_data_url"] = TEST_ZIP_FPATH + + # - Add fake metadata + _ = create_fake_metadata_file( + base_dir=base_dir, + product="RAW", + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + metadata_dict=metadata_dict, + ) + + runner = CliRunner() + runner.invoke( + disdrodb_download_archive, + ["--base_dir", base_dir], + ) diff --git a/disdrodb/tests/test_data_transfer/test_upload_data.py b/disdrodb/tests/test_data_transfer/test_upload_data.py index 2ffe470c..f71e65d2 100644 --- a/disdrodb/tests/test_data_transfer/test_upload_data.py +++ b/disdrodb/tests/test_data_transfer/test_upload_data.py @@ -88,7 +88,7 @@ def test_upload_station(tmp_path, requests_mock, mocker, station_url, force, pla ) # Define token name - if "sandbox.zenodo": + if platform == "sandbox.zenodo": token_key = "zenodo_sandbox_token" else: token_key = "zenodo_token" @@ -129,6 +129,50 @@ def test_upload_station(tmp_path, requests_mock, mocker, station_url, force, pla new_station_url == "dummy_url" +def test_upload_with_invalid_platform(tmp_path, requests_mock, mocker): + """Test upload of station data.""" + base_dir = tmp_path / "DISDRODB" + data_source = "test_data_source" + campaign_name = "test_campaign_name" + station_name = "test_station_name" + + force = True + metadata_dict = {} + metadata_dict["disdrodb_data_url"] = "existing_url" + + _ = create_fake_metadata_file( + base_dir=base_dir, + metadata_dict=metadata_dict, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + _ = create_fake_raw_data_file( + base_dir=base_dir, data_source=data_source, campaign_name=campaign_name, station_name=station_name + ) + + # Check it raise error if invalid platform + with pytest.raises(NotImplementedError): + upload_station( + platform="invalid_platform", + base_dir=str(base_dir), + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + force=force, + ) + + with pytest.raises(NotImplementedError): + upload_archive( + platform="invalid_platform", + base_dir=str(base_dir), + data_sources=data_source, + campaign_names=campaign_name, + station_names=station_name, + force=force, + ) + + @pytest.mark.parametrize("force", [True, False]) @pytest.mark.parametrize("station_url", ["existing_url", ""]) @pytest.mark.parametrize("platform", ["sandbox.zenodo", "zenodo"]) @@ -155,7 +199,7 @@ def test_upload_archive(tmp_path, requests_mock, mocker, station_url, force, pla ) # Define token name - if "sandbox.zenodo": + if platform == "sandbox.zenodo": token_key = "zenodo_sandbox_token" else: token_key = "zenodo_token" @@ -188,3 +232,37 @@ def test_upload_archive(tmp_path, requests_mock, mocker, station_url, force, pla else: # Upload --> key update assert new_station_url == "dummy_url" + + +@pytest.mark.parametrize("platform", ["sandbox.zenodo", "zenodo"]) +def test_upload_archive_do_not_stop(tmp_path, requests_mock, mocker, platform): + """Test upload of archive stations is not stopped by station errors.""" + base_dir = tmp_path / "DISDRODB" + data_source = "test_data_source" + campaign_name = "test_campaign_name" + station_name = "test_station_name" + + metadata_dict = {} + metadata_dict["disdrodb_data_url"] = "dummy_url" + + _ = create_fake_metadata_file( + base_dir=base_dir, + metadata_dict=metadata_dict, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + + _ = create_fake_raw_data_file( + base_dir=base_dir, data_source=data_source, campaign_name=campaign_name, station_name=station_name + ) + + mocker.patch("disdrodb.data_transfer.upload_data.upload_station", side_effect=Exception("Whatever error occurred")) + upload_archive( + platform=platform, + base_dir=str(base_dir), + data_sources=data_source, + campaign_names=campaign_name, + station_names=station_name, + force=False, + ) diff --git a/disdrodb/tests/test_data_transfer/test_zenodo.py b/disdrodb/tests/test_data_transfer/test_zenodo.py new file mode 100644 index 00000000..859c601b --- /dev/null +++ b/disdrodb/tests/test_data_transfer/test_zenodo.py @@ -0,0 +1,185 @@ +# #!/usr/bin/env python3 + +# # -----------------------------------------------------------------------------. +# # Copyright (c) 2021-2023 DISDRODB developers +# # +# # This program is free software: you can redistribute it and/or modify +# # it under the terms of the GNU General Public License as published by +# # the Free Software Foundation, either version 3 of the License, or +# # (at your option) any later version. +# # +# # This program is distributed in the hope that it will be useful, +# # but WITHOUT ANY WARRANTY; without even the implied warranty of +# # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# # GNU General Public License for more details. +# # +# # You should have received a copy of the GNU General Public License +# # along with this program. If not, see . +# # -----------------------------------------------------------------------------. +"""Test DISDRODB zenodo utility.""" +import os +import pytest +from disdrodb.data_transfer.zenodo import ( + _check_http_response, + _define_disdrodb_data_url, + _define_creators_list, + upload_station_to_zenodo, +) + +class MockResponse: + """Create Mock Request Response.""" + + def __init__(self, status_code, json_data=None): + self.status_code = status_code + self._json_data = json_data or {} + + def json(self): + return self._json_data + + +class Test_Check_Http_Response: + """Test check_http_response behaviour.""" + + def test_check_http_response_success(self): + """Test case where the status code is as expected.""" + response = MockResponse(200) + try: + _check_http_response(response, 200, "test task") + except ValueError: + pytest.fail("Unexpected ValueError raised") + + def test_check_http_response_error_message(self): + """Test case where the status code is different and an error message is included.""" + response = MockResponse(400, {"message": "Bad request"}) + with pytest.raises(ValueError) as exc_info: + _check_http_response(response, 200, "test task") + assert "Error test task: 400 Bad request" in str(exc_info.value) + + def test_check_http_response_detailed_errors(self): + """Test case with detailed errors.""" + response = MockResponse(400, {"errors": [{"field": "title", "message": "Required"}]}) + with pytest.raises(ValueError) as exc_info: + _check_http_response(response, 200, "test task") + assert "Error test task: 400\n- title: Required" in str(exc_info.value) + + +def test_define_disdrodb_data_url(): + """Test Zenodo disdrodb_data_url.""" + zenodo_host = "zenodo.org" + deposit_id = "123456" + filename = "testfile.txt" + + expected_url = f"https://{zenodo_host}/records/{deposit_id}/files/{filename}?download=1" + actual_url = _define_disdrodb_data_url(zenodo_host, deposit_id, filename) + assert actual_url == expected_url, "URL does not match expected format" + + +class Test_Define_Creators_List: + """Test define_creators_list.""" + + def test_when_fields_number_correspond(self): + metadata = { + "authors": "John Doe; Jane Smith", + "authors_url": "http://example.com/john, http://example.com/jane", + "institution": "University A, University B", + } + expected_result = [ + {"name": "John Doe", "identifier": "http://example.com/john", "affiliation": "University A"}, + {"name": "Jane Smith", "identifier": "http://example.com/jane", "affiliation": "University B"} + ] + + assert _define_creators_list(metadata) == expected_result + + @pytest.mark.parametrize("key", ["authors", "authors_url", "institution"]) + def test_empty_key(self, key): + metadata = { + "authors": "John Doe", + "authors_url": "http://example.com/john", + "institution": "University A", + } + metadata[key] = "" + key_value_mapping = {"authors": "name", "authors_url": "identifier", "institution": "affiliation"} + expected_result = [ + {"name": "John Doe", "identifier": "http://example.com/john", "affiliation": "University A"}, + ] + expected_result[0][key_value_mapping[key]] = "" + assert _define_creators_list(metadata) == expected_result + + def test_all_empty_key(self): + metadata = { + "authors": "", + "authors_url": "", + "institution": "", + } + expected_result = [ + {"name": "", "identifier": "", "affiliation": ""}, + ] + assert _define_creators_list(metadata) == expected_result + + def test_institution_is_replicated(self): + metadata = { + "authors": "John Doe; Jane Smith", + "authors_url": "http://example.com/john, http://example.com/jane", + "institution": "University A", + } + expected_result = [ + {"name": "John Doe", "identifier": "http://example.com/john", "affiliation": "University A"}, + {"name": "Jane Smith", "identifier": "http://example.com/jane", "affiliation": "University A"} + ] + + assert _define_creators_list(metadata) == expected_result + + def test_identifiers_number_mismatch(self): + metadata = { + "authors": "John Doe; Jane Smith", + "authors_url": "http://example.com/john", + "institution": "University A", + } + expected_result = [ + {"name": "John Doe", "identifier": "", "affiliation": "University A"}, + {"name": "Jane Smith", "identifier": "", "affiliation": "University A"} + ] + assert _define_creators_list(metadata) == expected_result + + def test_institution_number_mismatch(self): + metadata = { + "authors": "John Doe; Jane Smith; Scooby Doo", + "authors_url": "http://example.com/john", + "institution": "University A, University B", + } + expected_result = [ + {"name": "John Doe", "identifier": "", "affiliation": ""}, + {"name": "Jane Smith", "identifier": "", "affiliation": ""}, + {"name": "Scooby Doo", "identifier": "", "affiliation": ""} + ] + assert _define_creators_list(metadata) == expected_result + + + def test_when_key_is_missing(self): + metadata = { + "authors": "John Doe;Jane Smith", + "institution": "University A, University B" + } + assert _define_creators_list(metadata) == [] + + +def test_if_upload_raise_error_remove_zip_file(tmp_path, mocker): + """Test that temporary zip file are removed if something fail !.""" + # Create a dummy file at station_zip_filepath + station_name = "40" + station_zip_fpath = str(tmp_path / f"{station_name}.zip") + with open(station_zip_fpath, "w") as f: + f.write("dummy content") + assert os.path.exists(station_zip_fpath) + + # Mock stuffs + mocker.patch("disdrodb.utils.compression._zip_dir", return_value=station_zip_fpath) + mocker.patch("disdrodb.data_transfer.zenodo._upload_file_to_zenodo", side_effect=Exception("Whatever error occurred")) + + # Test it remove the file if something fail + with pytest.raises(ValueError): + upload_station_to_zenodo(metadata_filepath=f"{station_name}.yml") + + assert not os.path.exists(station_zip_fpath) + + \ No newline at end of file diff --git a/disdrodb/tests/test_issue/test_issue.py b/disdrodb/tests/test_issue/test_issue_checks.py similarity index 65% rename from disdrodb/tests/test_issue/test_issue.py rename to disdrodb/tests/test_issue/test_issue_checks.py index 8172cd39..778008a0 100644 --- a/disdrodb/tests/test_issue/test_issue.py +++ b/disdrodb/tests/test_issue/test_issue_checks.py @@ -17,13 +17,8 @@ # along with this program. If not, see . # -----------------------------------------------------------------------------. """Check DISDRODB L0 issues processing.""" - -import os -from io import StringIO - -import numpy as np import pytest -import yaml +import numpy as np from disdrodb.issue.checks import ( _check_time_period_nested_list_format, @@ -37,8 +32,6 @@ check_time_periods, check_timesteps, ) -from disdrodb.issue.reader import read_issue -from disdrodb.issue.writer import _write_issue, _write_issue_docs ####--------------------------------------------------------------------------. #### Checks @@ -66,37 +59,6 @@ def test__is_numpy_array_string(): #### Writer -def test_write_issue_docs(): - # Create a mock file object - mock_file = StringIO() - - # Call the function under test - _write_issue_docs(mock_file) - - # Get the written data from the mock file object - written_data = mock_file.getvalue() - - # Check that the written data matches the expected output - expected_output = """# This file is used to store timesteps/time periods with wrong/corrupted observation. -# The specified timesteps are dropped during the L0 processing. -# The time format used is the isoformat : YYYY-mm-dd HH:MM:SS. -# The 'timesteps' key enable to specify the list of timesteps to be discarded. -# The 'time_period' key enable to specify the time periods to be dropped. -# Example: -# -# timesteps: -# - 2018-12-07 14:15:00 -# - 2018-12-07 14:17:00 -# - 2018-12-07 14:19:00 -# - 2018-12-07 14:25:00 -# time_period: -# - ['2018-08-01 12:00:00', '2018-08-01 14:00:00'] -# - ['2018-08-01 15:44:30', '2018-08-01 15:59:31'] -# - ['2018-08-02 12:44:30', '2018-08-02 12:59:31'] \n -""" - assert written_data == expected_output - - def test__is_numpy_array_datetime(): arr = np.array(["2022-01-01", "2022-01-02"], dtype="datetime64") assert _is_numpy_array_datetime(arr) is True @@ -105,48 +67,45 @@ def test__is_numpy_array_datetime(): assert _is_numpy_array_datetime(arr) is False -def test__check_timestep_datetime_accuracy(): - timesteps = np.array(["2022-01-01T01:00:00", "2022-01-01T02:00:00"], dtype="datetime64[s]") - assert np.array_equal(_check_timestep_datetime_accuracy(timesteps, unit="s"), timesteps) - - with pytest.raises(ValueError): - timesteps = np.array(["2022-01-01", "2022-01-02"], dtype="datetime64[D]") - _check_timestep_datetime_accuracy(timesteps, unit="s") - - -def test__check_timesteps_string(): - timesteps = ["2022-01-01 01:00:00", "2022-01-01 02:00:00"] - expected_output = np.array(["2022-01-01T01:00:00", "2022-01-01T02:00:00"], dtype="datetime64[s]") - assert np.array_equal(_check_timesteps_string(timesteps), expected_output) - - with pytest.raises(ValueError): - timesteps = ["2022-01-01 01:00", "2022-01-01 02:00:00"] - _check_timesteps_string(timesteps) - - def test_check_timesteps(): + """Check validity testing of timesteps.""" # Test None input assert check_timesteps(None) is None - # Test string input + # Test correct string input timesteps_string = "2022-01-01 01:00:00" expected_output_string = np.array(["2022-01-01T01:00:00"], dtype="datetime64[s]") assert np.array_equal(check_timesteps(timesteps_string), expected_output_string) - # Test list of string inputs + # Test correct list of string inputs timesteps_string_list = ["2022-01-01 01:00:00", "2022-01-01 02:00:00"] expected_output_string_list = np.array(["2022-01-01T01:00:00", "2022-01-01T02:00:00"], dtype="datetime64[s]") assert np.array_equal(check_timesteps(timesteps_string_list), expected_output_string_list) - # Test datetime input + # Test correct datetime input timesteps_datetime = np.array(["2022-01-01T01:00:00", "2022-01-01T02:00:00"], dtype="datetime64[s]") expected_output_datetime = np.array(["2022-01-01T01:00:00", "2022-01-01T02:00:00"], dtype="datetime64[s]") assert np.array_equal(check_timesteps(timesteps_datetime), expected_output_datetime) - # Test invalid input + # Test invalid type input with pytest.raises(TypeError): check_timesteps(123) + # Test invalid datetime input + with pytest.raises(ValueError): + timesteps = np.array(["2022-01-01", "2022-01-02"], dtype="datetime64[D]") + check_timesteps(timesteps) + + # Test invalid list of string (wrong temporal resolution) + with pytest.raises(ValueError): + timesteps = ["2022-01-01 01:00", "2022-01-01 02:00"] + check_timesteps(timesteps) + + # Test invalid list of string (wrong time format) + with pytest.raises(ValueError): + timesteps = ["2022-15-01 01:00:00", "2022-15-01 02:00:00"] + check_timesteps(timesteps) + def test_check_time_period_nested_list_format(): # Test valid input @@ -254,43 +213,4 @@ def test_check_issue_dict(): check_issue_dict(issue_dict) -def test_write_issue(tmpdir): - """Test the _write_issue function.""" - # Define test inputs - filepath = os.path.join(tmpdir, "test_yml") - timesteps = np.array([0, 1, 2]) - time_periods = np.array([[0, 1], [2, 3]]) - - # Call function - _write_issue(filepath, timesteps=timesteps, time_periods=time_periods) - - # Load YAML file - with open(filepath) as f: - issue_dict = yaml.load(f, Loader=yaml.FullLoader) - - # Check the issue dictionary - assert isinstance(issue_dict, dict) - assert len(issue_dict) == 2 - assert issue_dict.keys() == {"timesteps", "time_periods"} - assert np.array_equal(issue_dict["timesteps"], timesteps.astype(str).tolist()) - assert np.array_equal(issue_dict["time_periods"], time_periods.astype(str).tolist()) - - # Test dictionary with valid keys and timesteps - timesteps = ["2022-01-01 01:00:00", "2022-01-01 02:00:00"] - - issue_dict = { - "timesteps": timesteps, - } - - _write_issue(filepath, timesteps=np.array(timesteps), time_periods=None) - result = read_issue(filepath) - - timesteps_datetime = np.array(timesteps, dtype="datetime64[s]") - expected_result = { - "timesteps": timesteps_datetime, - "time_periods": None, - } - # assert np.array_equal(result,expected_result) - assert set(result.keys()) == set(expected_result.keys()) - assert np.array_equal(result["timesteps"], expected_result["timesteps"]) diff --git a/disdrodb/tests/test_issue/test_issue_writer.py b/disdrodb/tests/test_issue/test_issue_writer.py new file mode 100644 index 00000000..8f2aefb1 --- /dev/null +++ b/disdrodb/tests/test_issue/test_issue_writer.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 + +# -----------------------------------------------------------------------------. +# Copyright (c) 2021-2023 DISDRODB developers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# -----------------------------------------------------------------------------. +"""Test DISDRODB issue writer.""" +import os + +import pytest +import numpy as np +import yaml + +from io import StringIO +from disdrodb.issue.reader import read_station_issue, read_issue +from disdrodb.issue.writer import write_issue, _write_issue_docs, create_station_issue + + + +def test_write_issue_docs(): + """Test the writing of the issue YAML documentation.""" + # Create a mock file object + mock_file = StringIO() + + # Call the function under test + _write_issue_docs(mock_file) + + # Get the written data from the mock file object + written_data = mock_file.getvalue() + + # Check that the written data matches the expected output + expected_output = """# This file is used to store timesteps/time periods with wrong/corrupted observation. +# The specified timesteps are dropped during the L0 processing. +# The time format used is the isoformat : YYYY-mm-dd HH:MM:SS. +# The 'timesteps' key enable to specify the list of timesteps to be discarded. +# The 'time_period' key enable to specify the time periods to be dropped. +# Example: +# +# timesteps: +# - 2018-12-07 14:15:00 +# - 2018-12-07 14:17:00 +# - 2018-12-07 14:19:00 +# - 2018-12-07 14:25:00 +# time_period: +# - ['2018-08-01 12:00:00', '2018-08-01 14:00:00'] +# - ['2018-08-01 15:44:30', '2018-08-01 15:59:31'] +# - ['2018-08-02 12:44:30', '2018-08-02 12:59:31'] \n +""" + assert written_data == expected_output + + +def test_write_issue(tmpdir): + """Test the write_issue function.""" + # Define test inputs + filepath = os.path.join(tmpdir, "test_yml") + timesteps = np.array([0, 1, 2]) + time_periods = np.array([[0, 1], [2, 3]]) + + # Call function + write_issue(filepath, timesteps=timesteps, time_periods=time_periods) + + # Load YAML file + with open(filepath) as f: + issue_dict = yaml.load(f, Loader=yaml.FullLoader) + + # Check the issue dictionary + assert isinstance(issue_dict, dict) + assert len(issue_dict) == 2 + assert issue_dict.keys() == {"timesteps", "time_periods"} + assert np.array_equal(issue_dict["timesteps"], timesteps.astype(str).tolist()) + assert np.array_equal(issue_dict["time_periods"], time_periods.astype(str).tolist()) + + # Test dictionary with valid keys and timesteps + timesteps = ["2022-01-01 01:00:00", "2022-01-01 02:00:00"] + + issue_dict = { + "timesteps": timesteps, + } + + write_issue(filepath, timesteps=np.array(timesteps), time_periods=None) + + result = read_issue(filepath) + + timesteps_datetime = np.array(timesteps, dtype="datetime64[s]") + expected_result = { + "timesteps": timesteps_datetime, + "time_periods": None, + } + # assert np.array_equal(result,expected_result) + assert set(result.keys()) == set(expected_result.keys()) + assert np.array_equal(result["timesteps"], expected_result["timesteps"]) + + +def test_create_station_issue(tmp_path): + """Test the creation of the default issue YAML file.""" + base_dir = tmp_path / "DISDRODB" + data_source = "DATA_SOURCE" + campaign_name = "CAMPAIGN_NAME" + station_name = "station_name" + + _ = create_station_issue( + base_dir=base_dir, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + issue_dict = read_station_issue( + base_dir=base_dir, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + assert isinstance(issue_dict, dict) + issue_dict["timesteps"] = None + issue_dict["time_periods"] = None + + # Test it raise error if creating when already existing + with pytest.raises(ValueError): + create_station_issue( + base_dir=base_dir, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) \ No newline at end of file diff --git a/disdrodb/tests/test_l0/test_cmd_processing.py b/disdrodb/tests/test_l0/test_cmd_processing.py index dddcb5c3..f4fd8dd7 100644 --- a/disdrodb/tests/test_l0/test_cmd_processing.py +++ b/disdrodb/tests/test_l0/test_cmd_processing.py @@ -26,7 +26,13 @@ from disdrodb import __root_path__ from disdrodb.api.path import define_station_dir -from disdrodb.utils.directories import list_files +from disdrodb.l0.scripts.disdrodb_run_l0 import disdrodb_run_l0 +from disdrodb.l0.scripts.disdrodb_run_l0_station import disdrodb_run_l0_station +from disdrodb.l0.scripts.disdrodb_run_l0a import disdrodb_run_l0a +from disdrodb.l0.scripts.disdrodb_run_l0a_station import disdrodb_run_l0a_station +from disdrodb.l0.scripts.disdrodb_run_l0b import disdrodb_run_l0b +from disdrodb.l0.scripts.disdrodb_run_l0b_station import disdrodb_run_l0b_station +from disdrodb.utils.directories import count_files BASE_DIR = os.path.join(__root_path__, "disdrodb", "tests", "data", "check_readers", "DISDRODB") DATA_SOURCE = "EPFL" @@ -34,158 +40,171 @@ STATION_NAME = "10" -@pytest.fixture -def remove_processed_folder(request: list) -> None: - processed_folder = os.path.join(BASE_DIR, "Processed") - if os.path.exists(processed_folder): - shutil.rmtree(processed_folder) - yield - if os.path.exists(processed_folder): - shutil.rmtree(processed_folder) - - @pytest.mark.parametrize("parallel", [True, False]) -@pytest.mark.parametrize("remove_processed_folder", [()], indirect=True) -def test_disdrodb_run_l0a_station(remove_processed_folder, parallel): +def test_disdrodb_run_l0a_station(tmp_path, parallel): """Test the disdrodb_run_l0a_station command.""" - - from disdrodb.l0.scripts.disdrodb_run_l0a_station import disdrodb_run_l0a_station + test_base_dir = tmp_path / "DISDRODB" + shutil.copytree(BASE_DIR, test_base_dir) runner = CliRunner() runner.invoke( disdrodb_run_l0a_station, - [DATA_SOURCE, CAMPAIGN_NAME, STATION_NAME, "--base_dir", BASE_DIR, "--parallel", parallel], + [DATA_SOURCE, CAMPAIGN_NAME, STATION_NAME, "--base_dir", test_base_dir, "--parallel", parallel], ) product = "L0A" station_dir = define_station_dir( - base_dir=BASE_DIR, + base_dir=test_base_dir, product=product, data_source=DATA_SOURCE, campaign_name=CAMPAIGN_NAME, station_name=STATION_NAME, ) - filepaths = list_files(station_dir, glob_pattern="*.parquet", recursive=True) - assert len(filepaths) > 0 + assert count_files(station_dir, glob_pattern="*.parquet", recursive=True) > 0 @pytest.mark.parametrize("parallel", [True, False]) -@pytest.mark.parametrize("remove_processed_folder", [()], indirect=True) -def test_disdrodb_run_l0b_station(remove_processed_folder, parallel): +def test_disdrodb_run_l0b_station(tmp_path, parallel): """Test the disdrodb_run_l0b_station command.""" - from disdrodb.l0.scripts.disdrodb_run_l0a_station import disdrodb_run_l0a_station + test_base_dir = tmp_path / "DISDRODB" + shutil.copytree(BASE_DIR, test_base_dir) runner = CliRunner() runner.invoke( disdrodb_run_l0a_station, - [DATA_SOURCE, CAMPAIGN_NAME, STATION_NAME, "--base_dir", BASE_DIR, "--parallel", parallel], + [DATA_SOURCE, CAMPAIGN_NAME, STATION_NAME, "--base_dir", test_base_dir, "--parallel", parallel], ) - from disdrodb.l0.scripts.disdrodb_run_l0b_station import disdrodb_run_l0b_station - runner.invoke( disdrodb_run_l0b_station, - [DATA_SOURCE, CAMPAIGN_NAME, STATION_NAME, "--base_dir", BASE_DIR, "--parallel", parallel], + [DATA_SOURCE, CAMPAIGN_NAME, STATION_NAME, "--base_dir", test_base_dir, "--parallel", parallel], ) product = "L0B" station_dir = define_station_dir( - base_dir=BASE_DIR, + base_dir=test_base_dir, product=product, data_source=DATA_SOURCE, campaign_name=CAMPAIGN_NAME, station_name=STATION_NAME, ) - filepaths = list_files(station_dir, glob_pattern="*.nc", recursive=True) - assert len(filepaths) > 0 + assert count_files(station_dir, glob_pattern="*.nc", recursive=True) > 0 -@pytest.mark.parametrize("parallel", [True, False]) -@pytest.mark.parametrize("remove_processed_folder", [()], indirect=True) -def test_disdrodb_run_l0_station(remove_processed_folder, parallel): +@pytest.mark.parametrize("verbose", [True, False]) +def test_disdrodb_run_l0_station(tmp_path, verbose): """Test the disdrodb_run_l0_station command.""" - - from disdrodb.l0.scripts.disdrodb_run_l0_station import disdrodb_run_l0_station + test_base_dir = tmp_path / "DISDRODB" + shutil.copytree(BASE_DIR, test_base_dir) runner = CliRunner() runner.invoke( disdrodb_run_l0_station, - [DATA_SOURCE, CAMPAIGN_NAME, STATION_NAME, "--base_dir", BASE_DIR, "--parallel", parallel], + [DATA_SOURCE, CAMPAIGN_NAME, STATION_NAME, "--base_dir", test_base_dir, "--verbose", verbose], ) product = "L0B" station_dir = define_station_dir( - base_dir=BASE_DIR, + base_dir=test_base_dir, product=product, data_source=DATA_SOURCE, campaign_name=CAMPAIGN_NAME, station_name=STATION_NAME, ) - filepaths = list_files(station_dir, glob_pattern="*.nc", recursive=True) - assert len(filepaths) > 0 + assert count_files(station_dir, glob_pattern="*.nc", recursive=True) > 0 -@pytest.mark.parametrize("remove_processed_folder", [()], indirect=True) -def test_disdrodb_run_l0a(remove_processed_folder): +def test_disdrodb_run_l0a(tmp_path): """Test the disdrodb_run_l0a command.""" - - from disdrodb.l0.scripts.disdrodb_run_l0a import disdrodb_run_l0a + test_base_dir = tmp_path / "DISDRODB" + shutil.copytree(BASE_DIR, test_base_dir) runner = CliRunner() - runner.invoke(disdrodb_run_l0a, ["--base_dir", BASE_DIR]) + runner.invoke(disdrodb_run_l0a, ["--base_dir", test_base_dir]) product = "L0A" station_dir = define_station_dir( - base_dir=BASE_DIR, + base_dir=test_base_dir, product=product, data_source=DATA_SOURCE, campaign_name=CAMPAIGN_NAME, station_name=STATION_NAME, ) - filepaths = list_files(station_dir, glob_pattern="*.parquet", recursive=True) - assert len(filepaths) > 0 + assert count_files(station_dir, glob_pattern="*.parquet", recursive=True) > 0 -@pytest.mark.parametrize("remove_processed_folder", [()], indirect=True) -def test_disdrodb_run_l0b(remove_processed_folder): +def test_disdrodb_run_l0b(tmp_path): """Test the disdrodb_run_l0b command.""" - - from disdrodb.l0.scripts.disdrodb_run_l0a import disdrodb_run_l0a + test_base_dir = tmp_path / "DISDRODB" + shutil.copytree(BASE_DIR, test_base_dir) runner = CliRunner() - runner.invoke(disdrodb_run_l0a, ["--base_dir", BASE_DIR]) + runner.invoke(disdrodb_run_l0a, ["--base_dir", test_base_dir]) - from disdrodb.l0.scripts.disdrodb_run_l0b import disdrodb_run_l0b - - runner.invoke(disdrodb_run_l0b, ["--base_dir", BASE_DIR]) + runner.invoke(disdrodb_run_l0b, ["--base_dir", test_base_dir]) product = "L0B" station_dir = define_station_dir( - base_dir=BASE_DIR, + base_dir=test_base_dir, product=product, data_source=DATA_SOURCE, campaign_name=CAMPAIGN_NAME, station_name=STATION_NAME, ) - filepaths = list_files(station_dir, glob_pattern="*.nc", recursive=True) - assert len(filepaths) > 0 + assert count_files(station_dir, glob_pattern="*.nc", recursive=True) > 0 -@pytest.mark.parametrize("remove_processed_folder", [()], indirect=True) -def test_full_disdrodb_run_l0(remove_processed_folder): +@pytest.mark.parametrize("remove_l0a", [True, False]) +@pytest.mark.parametrize("remove_l0b", [True, False]) +@pytest.mark.parametrize("l0b_concat", [True, False]) +def test_disdrodb_run_l0(tmp_path, remove_l0a, remove_l0b, l0b_concat): """Test the disdrodb_run_l0b command.""" - - from disdrodb.l0.scripts.disdrodb_run_l0 import disdrodb_run_l0 + test_base_dir = tmp_path / "DISDRODB" + shutil.copytree(BASE_DIR, test_base_dir) runner = CliRunner() - runner.invoke(disdrodb_run_l0, ["--base_dir", BASE_DIR]) + runner.invoke( + disdrodb_run_l0, + [ + "--base_dir", + test_base_dir, + "--debugging_mode", + True, + "--remove_l0a", + remove_l0a, + "--remove_l0b", + remove_l0b, + "--l0b_concat", + l0b_concat, + ], + ) - product = "L0B" - station_dir = define_station_dir( - base_dir=BASE_DIR, - product=product, + l0a_station_dir = define_station_dir( + base_dir=test_base_dir, + product="L0A", + data_source=DATA_SOURCE, + campaign_name=CAMPAIGN_NAME, + station_name=STATION_NAME, + ) + l0b_station_dir = define_station_dir( + base_dir=test_base_dir, + product="L0B", data_source=DATA_SOURCE, campaign_name=CAMPAIGN_NAME, station_name=STATION_NAME, ) - filepaths = list_files(station_dir, glob_pattern="*.nc", recursive=True) - assert len(filepaths) > 0 + if remove_l0a: + assert count_files(l0a_station_dir, glob_pattern="*.parquet", recursive=True) == 0 + + if not remove_l0a: + assert count_files(l0a_station_dir, glob_pattern="*.parquet", recursive=True) > 0 + + if l0b_concat: + if remove_l0b: + assert count_files(l0b_station_dir, glob_pattern="*.nc", recursive=True) == 0 + else: + assert count_files(l0b_station_dir, glob_pattern="*.nc", recursive=True) > 0 + + # If not L0B concat, do not remove L0B also if remove_l0b is specified ! + if not l0b_concat: + if remove_l0b: + assert count_files(l0b_station_dir, glob_pattern="*.nc", recursive=True) > 0 diff --git a/disdrodb/tests/test_l0/test_l0b_concat.py b/disdrodb/tests/test_l0/test_l0b_concat.py index ee5bbfac..d54088d8 100644 --- a/disdrodb/tests/test_l0/test_l0b_concat.py +++ b/disdrodb/tests/test_l0/test_l0b_concat.py @@ -22,13 +22,14 @@ import numpy as np import pandas as pd +import pytest import xarray as xr from disdrodb.api.path import define_campaign_dir -from disdrodb.l0.l0_processing import run_l0b_concat +from disdrodb.l0.l0_processing import run_l0b_concat, run_l0b_concat_station from disdrodb.l0.routines import run_disdrodb_l0b_concat from disdrodb.tests.conftest import create_fake_metadata_file, create_fake_station_dir -from disdrodb.utils.directories import list_files +from disdrodb.utils.directories import count_files, list_files from disdrodb.utils.netcdf import xr_concat_datasets @@ -160,7 +161,7 @@ def test_xr_concat_completely_partial_overlapped_datasets(tmp_path): np.testing.assert_allclose(time_values.astype(float), unique_time_data.astype(float)) -def test_run_l0b_concat_station(tmp_path): +def test_run_l0b_concat(tmp_path): # Define station info base_dir = tmp_path / "DISDRODB" data_source = "DATA_SOURCE" @@ -198,7 +199,7 @@ def mock_write_l0b(ds: xr.Dataset, filepath: str, force=False) -> None: l0b_processing.write_l0b = mock_write_l0b # Run concatenation command - run_l0b_concat(processed_dir=processed_dir, station_name=station_name, remove=False, verbose=False) + run_l0b_concat(processed_dir=processed_dir, station_name=station_name, verbose=False) # Assert only 1 file is created list_concatenated_files = list_files(os.path.join(processed_dir, "L0B"), glob_pattern="*.nc", recursive=False) @@ -209,6 +210,68 @@ def mock_write_l0b(ds: xr.Dataset, filepath: str, force=False) -> None: assert len(ds["time"].values) == 6 +def test_run_l0b_concat_station(tmp_path): + # Define stations info + base_dir = tmp_path / "DISDRODB" + data_source = "DATA_SOURCE" + campaign_name = "CAMPAIGN_NAME" + station_name1 = "test_station_1" + + # Define fake directory structure for the two L0B stations + # # Define fake L0B directory structure + station1_dir = create_fake_station_dir( + base_dir=base_dir, + product="L0B", + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name1, + ) + _ = create_fake_metadata_file( + base_dir=base_dir, + product="L0B", + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name1, + ) + + # Add dummy L0B files for two stations + filepath1 = os.path.join(station1_dir, f"{station_name1}_file.nc") + time_data_1 = np.array([0.0, 1.0, 2.0], dtype=np.float64) + + _ = create_dummy_l0b_file(filepath=filepath1, time=time_data_1) + + # Run concatenation command + run_l0b_concat_station( + base_dir=str(base_dir), + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name1, + remove_l0b=True, + verbose=False, + ) + + # Assert files where removed + assert not os.path.exists(filepath1) + + # Assert the presence of 2 concatenated netcdf files (one for each station) + processed_dir = define_campaign_dir( + base_dir=base_dir, product="L0B", data_source=data_source, campaign_name=campaign_name + ) + + assert count_files(os.path.join(processed_dir, "L0B"), glob_pattern="*.nc", recursive=False) == 1 + + # Check that if L0B files are removed, raise error if no stations available + with pytest.raises(ValueError): + run_l0b_concat_station( + base_dir=str(base_dir), + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name1, + remove_l0b=True, + verbose=False, + ) + + def test_run_disdrodb_l0b_concat(tmp_path): # Define stations info base_dir = tmp_path / "DISDRODB" @@ -267,26 +330,24 @@ def test_run_disdrodb_l0b_concat(tmp_path): verbose=False, ) - # BUGGY WITH PYTEST ! - # # Assert files where removed - # assert not os.path.exists(filepath1) - # assert not os.path.exists(filepath2) + # Assert files where removed + assert not os.path.exists(filepath1) + assert not os.path.exists(filepath2) - # # Assert the presence of 2 concatenated netcdf files (one for each station) - # processed_dir = define_campaign_dir( - # base_dir=base_dir, product="L0B", data_source=data_source, campaign_name=campaign_name - # ) + # Assert the presence of 2 concatenated netcdf files (one for each station) + processed_dir = define_campaign_dir( + base_dir=base_dir, product="L0B", data_source=data_source, campaign_name=campaign_name + ) - # list_concatenated_files = list_files(os.path.join(processed_dir, "L0B"), glob_pattern="*.nc", recursive=False) - # assert len(list_concatenated_files) == 2 + assert count_files(os.path.join(processed_dir, "L0B"), glob_pattern="*.nc", recursive=False) == 2 # Check that if L0B files are removed, raise error if no stations available - # with pytest.raises(ValueError): - # run_disdrodb_l0b_concat( - # base_dir=str(base_dir), - # data_sources=data_source, - # campaign_names=campaign_name, - # station_names=[station_name1, station_name2], - # remove_l0b=True, - # verbose=False, - # ) + with pytest.raises(ValueError): + run_disdrodb_l0b_concat( + base_dir=str(base_dir), + data_sources=data_source, + campaign_names=campaign_name, + station_names=[station_name1, station_name2], + remove_l0b=True, + verbose=False, + ) diff --git a/disdrodb/tests/test_metadata/test_metadata_checks.py b/disdrodb/tests/test_metadata/test_metadata_checks.py index 0445cc00..aa0f070a 100644 --- a/disdrodb/tests/test_metadata/test_metadata_checks.py +++ b/disdrodb/tests/test_metadata/test_metadata_checks.py @@ -24,6 +24,7 @@ from disdrodb.api.configs import available_sensor_names from disdrodb.l0.l0_reader import available_readers from disdrodb.metadata.checks import ( + _check_metadata_values, check_archive_metadata_campaign_name, check_archive_metadata_compliance, check_archive_metadata_data_source, @@ -38,6 +39,7 @@ ) from disdrodb.metadata.standards import get_valid_metadata_keys from disdrodb.tests.conftest import create_fake_metadata_file +from disdrodb.utils.yaml import read_yaml, write_yaml def test_check_metadata_geolocation(): @@ -86,14 +88,14 @@ def test_check_metadata_geolocation(): assert check_metadata_geolocation(metadata) is None -def test_identify_missing_metadata_keys(tmp_path, capsys): +def test_identify_empty_metadata_keys(tmp_path, capsys): base_dir = tmp_path / "DISDRODB" metadata_dict = {"key1": "value1"} metadata_filepath = create_fake_metadata_file(base_dir, metadata_dict=metadata_dict) # Test the key is empty -> print statement with the key name tested_key = "key2" - identify_empty_metadata_keys([metadata_filepath], [tested_key]) + identify_empty_metadata_keys([metadata_filepath], keys=tested_key) captured = capsys.readouterr() assert tested_key in str(captured.out) @@ -122,6 +124,22 @@ def test_check_archive_metadata_keys(tmp_path): is_valid = check_archive_metadata_keys(str(base_dir)) assert not is_valid + # Test 3 : Check missing metadata key + metadata_dict = {} + metadata_filepath = create_fake_metadata_file(base_dir=base_dir, metadata_dict=metadata_dict) + metadata_dict = read_yaml(metadata_filepath) + metadata_dict.pop("data_source") + write_yaml(metadata_dict, metadata_filepath) + is_valid = check_archive_metadata_keys(str(base_dir)) + assert not is_valid + + +def test_check_archive_metadata_valid_values(tmp_path): + """Test that None is an invalid value.""" + metadata_dict = {"key_is_None": None} + with pytest.raises(ValueError): + _check_metadata_values(metadata_dict) + def test_check_archive_metadata_campaign_name(tmp_path): base_dir = tmp_path / "DISDRODB" @@ -133,13 +151,32 @@ def test_check_archive_metadata_campaign_name(tmp_path): is_valid = check_archive_metadata_campaign_name(str(base_dir)) assert is_valid - # Test 2 : Wrong campaign_name metadata key + # Test 2 : Empty campaign_name campaign_name = "CAMPAIGN_NAME" metadata_dict = {"campaign_name": ""} _ = create_fake_metadata_file(base_dir=base_dir, campaign_name=campaign_name, metadata_dict=metadata_dict) is_valid = check_archive_metadata_campaign_name(str(base_dir)) assert not is_valid + # Test 3 : Wrong campaign_name + campaign_name = "CAMPAIGN_NAME" + metadata_dict = {"campaign_name": "ANOTHER_CAMPAIGN_NAME"} + _ = create_fake_metadata_file(base_dir=base_dir, campaign_name=campaign_name, metadata_dict=metadata_dict) + is_valid = check_archive_metadata_campaign_name(str(base_dir)) + assert not is_valid + + # Test 4 : Missing campaign_name + campaign_name = "CAMPAIGN_NAME" + metadata_filepath = create_fake_metadata_file( + base_dir=base_dir, + campaign_name=campaign_name, + ) + metadata_dict = read_yaml(metadata_filepath) + metadata_dict.pop("campaign_name", None) + write_yaml(metadata_dict, metadata_filepath) + is_valid = check_archive_metadata_campaign_name(str(base_dir)) + assert not is_valid + def test_check_archive_metadata_data_source(tmp_path): base_dir = tmp_path / "DISDRODB" @@ -151,13 +188,29 @@ def test_check_archive_metadata_data_source(tmp_path): is_valid = check_archive_metadata_data_source(str(base_dir)) assert is_valid - # Test 2 : Wrong data_source metadata key + # Test 2 : Empty data_source metadata key data_source = "DATA_SOURCE" metadata_dict = {"data_source": ""} _ = create_fake_metadata_file(base_dir=base_dir, data_source=data_source, metadata_dict=metadata_dict) is_valid = check_archive_metadata_data_source(str(base_dir)) assert not is_valid + # Test 3 : Wrong data_source + data_source = "DATA_SOURCE" + metadata_dict = {"data_source": "ANOTHER_DATA_SOURCE"} + _ = create_fake_metadata_file(base_dir=base_dir, data_source=data_source, metadata_dict=metadata_dict) + is_valid = check_archive_metadata_data_source(str(base_dir)) + assert not is_valid + + # Test 4 : Missing data_source + data_source = "DATA_SOURCE" + metadata_filepath = create_fake_metadata_file(base_dir=base_dir, data_source=data_source) + metadata_dict = read_yaml(metadata_filepath) + metadata_dict.pop("data_source", None) + write_yaml(metadata_dict, metadata_filepath) + is_valid = check_archive_metadata_data_source(str(base_dir)) + assert not is_valid + @pytest.mark.parametrize("sensor_name", available_sensor_names(product="L0A")) def test_check_archive_metadata_sensor_name(tmp_path, sensor_name): @@ -186,12 +239,28 @@ def test_check_archive_metadata_station_name(tmp_path): is_valid = check_archive_metadata_station_name(str(base_dir)) assert is_valid - # Test 2 : Wrong station_name metadata key + # Test 2 : Empty station_name metadata key metadata_dict = {"station_name": ""} _ = create_fake_metadata_file(base_dir=base_dir, metadata_dict=metadata_dict, station_name=station_name) is_valid = check_archive_metadata_station_name(str(base_dir)) assert not is_valid + # Test 3 : Wrong station_name + station_name = "STATION_NAME" + metadata_dict = {"station_name": "ANOTHER_STATION_NAME"} + _ = create_fake_metadata_file(base_dir=base_dir, station_name=station_name, metadata_dict=metadata_dict) + is_valid = check_archive_metadata_station_name(str(base_dir)) + assert not is_valid + + # Test 4 : Missing station_name + station_name = "STATION_NAME" + metadata_filepath = create_fake_metadata_file(base_dir=base_dir, station_name=station_name) + metadata_dict = read_yaml(metadata_filepath) + metadata_dict.pop("station_name", None) + write_yaml(metadata_dict, metadata_filepath) + is_valid = check_archive_metadata_station_name(str(base_dir)) + assert not is_valid + def test_check_archive_metadata_reader(tmp_path): base_dir = tmp_path / "DISDRODB" @@ -225,6 +294,7 @@ def test_check_archive_metadata_compliance(tmp_path): # We check only the failure, the success are tested in the above tests. metadata_dict = {"reader": ""} _ = create_fake_metadata_file(base_dir=base_dir, metadata_dict=metadata_dict) + # Test does not raise error ! result = check_archive_metadata_compliance(str(base_dir), raise_error=False) assert result is False diff --git a/disdrodb/tests/test_metadata/test_metadata_reader.py b/disdrodb/tests/test_metadata/test_metadata_reader.py new file mode 100644 index 00000000..3bb57f01 --- /dev/null +++ b/disdrodb/tests/test_metadata/test_metadata_reader.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 + +# -----------------------------------------------------------------------------. +# Copyright (c) 2021-2023 DISDRODB developers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# -----------------------------------------------------------------------------. +"""Test DISDRODB metadata reader.""" + +import pytest + +from disdrodb.metadata.reader import read_station_metadata +from disdrodb.tests.conftest import create_fake_metadata_file + + +@pytest.mark.parametrize("product", ["RAW", "L0A", "L0B"]) +def test_read_station_metadata(tmp_path, product): + base_dir = tmp_path / "DISDRODB" + data_source = "DATA_SOURCE" + campaign_name = "CAMPAIGN_NAME" + station_name = "station_name" + + _ = create_fake_metadata_file( + base_dir=base_dir, + product=product, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + metadata_dict = read_station_metadata( + base_dir=base_dir, + product=product, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + + assert isinstance(metadata_dict, dict) diff --git a/disdrodb/tests/test_metadata/test_metadata_io.py b/disdrodb/tests/test_metadata/test_metadata_search.py similarity index 65% rename from disdrodb/tests/test_metadata/test_metadata_io.py rename to disdrodb/tests/test_metadata/test_metadata_search.py index b86d0113..052ba520 100644 --- a/disdrodb/tests/test_metadata/test_metadata_io.py +++ b/disdrodb/tests/test_metadata/test_metadata_search.py @@ -18,28 +18,17 @@ # -----------------------------------------------------------------------------. """Test DISDRODB API metadata utility.""" -import os - import pytest -from disdrodb.api.path import define_metadata_filepath -from disdrodb.metadata.reader import read_station_metadata from disdrodb.metadata.search import ( _get_list_all_metadata, _get_list_metadata_with_data, get_list_metadata, ) -from disdrodb.metadata.writer import ( - create_station_metadata, - get_default_metadata_dict, - write_default_metadata, -) from disdrodb.tests.conftest import ( - create_fake_metadata_directory, create_fake_metadata_file, create_fake_raw_data_file, ) -from disdrodb.utils.yaml import read_yaml def test__get_list_all_metadata(tmp_path): @@ -203,88 +192,3 @@ def test_get_list_metadata_file(tmp_path): assert [metadata_filepath] == result -def test_get_default_metadata(): - assert isinstance(get_default_metadata_dict(), dict) - - -def test_write_default_metadata(tmp_path): - base_dir = tmp_path / "DISDRODB" - data_source = "DATA_SOURCE" - campaign_name = "CAMPAIGN_NAME" - station_name = "station_name" - - # Create the default metadata file - _ = create_fake_metadata_directory( - base_dir=base_dir, product="RAW", data_source=data_source, campaign_name=campaign_name - ) - metadata_filepath = define_metadata_filepath( - base_dir=base_dir, - product="RAW", - data_source=data_source, - campaign_name=campaign_name, - station_name=station_name, - check_exists=False, - ) - write_default_metadata(metadata_filepath) - - # Check file exist - assert os.path.exists(metadata_filepath) - - # Open it - dictionary = read_yaml(str(metadata_filepath)) - - # Check is the expected dictionary - expected_dict = get_default_metadata_dict() - expected_dict["data_source"] = data_source - expected_dict["campaign_name"] = campaign_name - expected_dict["station_name"] = station_name - assert expected_dict == dictionary - - -@pytest.mark.parametrize("product", ["RAW", "L0A", "L0B"]) -def test_read_station_metadata(tmp_path, product): - base_dir = tmp_path / "DISDRODB" - data_source = "DATA_SOURCE" - campaign_name = "CAMPAIGN_NAME" - station_name = "station_name" - - _ = create_fake_metadata_file( - base_dir=base_dir, - product=product, - data_source=data_source, - campaign_name=campaign_name, - station_name=station_name, - ) - metadata_dict = read_station_metadata( - base_dir=base_dir, - product=product, - data_source=data_source, - campaign_name=campaign_name, - station_name=station_name, - ) - - assert isinstance(metadata_dict, dict) - - -@pytest.mark.parametrize("product", ["RAW", "L0A", "L0B"]) -def test_create_station_metadata(tmp_path, product): - base_dir = tmp_path / "DISDRODB" - data_source = "DATA_SOURCE" - campaign_name = "CAMPAIGN_NAME" - station_name = "station_name" - - _ = create_station_metadata( - base_dir=base_dir, - product=product, - data_source=data_source, - campaign_name=campaign_name, - station_name=station_name, - ) - metadata_dict = read_station_metadata( - base_dir=base_dir, - product=product, - data_source=data_source, - campaign_name=campaign_name, - station_name=station_name, - ) - assert isinstance(metadata_dict, dict) diff --git a/disdrodb/tests/test_metadata/test_metadata_writer.py b/disdrodb/tests/test_metadata/test_metadata_writer.py new file mode 100644 index 00000000..2ad8428e --- /dev/null +++ b/disdrodb/tests/test_metadata/test_metadata_writer.py @@ -0,0 +1,66 @@ +#!/usr/bin/env python3 + +# -----------------------------------------------------------------------------. +# Copyright (c) 2021-2023 DISDRODB developers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# -----------------------------------------------------------------------------. +"""Test DISDRODB metadata writer.""" + +import pytest + +from disdrodb.metadata.reader import read_station_metadata +from disdrodb.metadata.writer import ( + create_station_metadata, + get_default_metadata_dict, +) + + +def test_get_default_metadata(): + assert isinstance(get_default_metadata_dict(), dict) + + +def test_create_station_metadata(tmp_path): + base_dir = tmp_path / "DISDRODB" + data_source = "DATA_SOURCE" + campaign_name = "CAMPAIGN_NAME" + station_name = "station_name" + product = "RAW" + + _ = create_station_metadata( + base_dir=base_dir, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + metadata_dict = read_station_metadata( + base_dir=base_dir, + product=product, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + assert isinstance(metadata_dict, dict) + metadata_dict["data_source"] = data_source + metadata_dict["campaign_name"] = campaign_name + metadata_dict["station_name"] = station_name + + # Test it raise error if creating when already existing + with pytest.raises(ValueError): + create_station_metadata( + base_dir=base_dir, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) diff --git a/disdrodb/utils/logger.py b/disdrodb/utils/logger.py index 0d81a759..91b38f5c 100644 --- a/disdrodb/utils/logger.py +++ b/disdrodb/utils/logger.py @@ -207,9 +207,10 @@ def log_debug(logger: logger, msg: str, verbose: bool = False) -> None: Whether to verbose the processing. The default is False. """ - logger.debug(msg) - if verbose: - print(" - " + msg) + if not os.environ.get("PYTEST_CURRENT_TEST"): + logger.debug(msg) + if verbose: + print(" - " + msg) def log_info(logger: logger, msg: str, verbose: bool = False) -> None: @@ -244,9 +245,10 @@ def log_warning(logger: logger, msg: str, verbose: bool = False) -> None: Whether to verbose the processing. The default is False. """ - logger.warning(msg) - if verbose: - print(" - " + msg) + if not os.environ.get("PYTEST_CURRENT_TEST"): + logger.warning(msg) + if verbose: + print(" - " + msg) def log_error(logger: logger, msg: str, verbose: bool = False) -> None: @@ -262,6 +264,7 @@ def log_error(logger: logger, msg: str, verbose: bool = False) -> None: Whether to verbose the processing. The default is False. """ - logger.error(msg) - if verbose: - print(" - " + msg) + if not os.environ.get("PYTEST_CURRENT_TEST"): + logger.error(msg) + if verbose: + print(" - " + msg)