diff --git a/.codebeatignore b/.codebeatignore new file mode 100644 index 00000000..75f142c2 --- /dev/null +++ b/.codebeatignore @@ -0,0 +1 @@ +disdrodb/l0/readers/* \ No newline at end of file diff --git a/disdrodb/api/check_directories.py b/disdrodb/api/check_directories.py index 92ad5689..df3ea1a0 100644 --- a/disdrodb/api/check_directories.py +++ b/disdrodb/api/check_directories.py @@ -225,7 +225,7 @@ def _check_presence_metadata_directory(raw_dir): If the 'metadata' does not exists, it create default metadata files for each station present in the 'data' directory. """ - from disdrodb.metadata.io import write_default_metadata + from disdrodb.metadata.writer import write_default_metadata if not _is_metadata_directory_available(raw_dir): # Create metadata directory @@ -268,7 +268,7 @@ def _check_presence_all_metadata_files(raw_dir): The function raise error if there is not a metadata file for each station directory present in the 'data' directory. """ - from disdrodb.metadata.io import write_default_metadata + from disdrodb.metadata.writer import write_default_metadata # Get stations with available metadata list_metadata_station_name = _get_available_stations_with_metadata_files(raw_dir) diff --git a/disdrodb/api/create_directories.py b/disdrodb/api/create_directories.py index 95d9eaa9..04757144 100644 --- a/disdrodb/api/create_directories.py +++ b/disdrodb/api/create_directories.py @@ -27,6 +27,7 @@ from disdrodb.api.info import infer_campaign_name_from_path, infer_data_source_from_path from disdrodb.api.path import ( + define_campaign_dir, define_issue_dir, define_issue_filepath, define_metadata_dir, @@ -208,6 +209,7 @@ def create_issue_directory(base_dir, data_source, campaign_name): return str(issue_dir) +#### L0A processing def create_initial_directory_structure( raw_dir, processed_dir, @@ -258,6 +260,7 @@ def create_initial_directory_structure( create_required_directory(os.path.join(processed_dir, product), dir_name=station_name) +#### > L0A processing def create_directory_structure(processed_dir, product, station_name, force, verbose=False): """Create directory structure for L0B and higher DISDRODB products.""" from disdrodb.api.check_directories import check_presence_metadata_file, check_processed_dir @@ -295,29 +298,116 @@ def create_directory_structure(processed_dir, product, station_name, force, verb ) -def create_test_archive(test_base_dir, data_source, campaign_name, station_name, force=False): +#### Initialization +def _create_station_directories( + data_source, + campaign_name, + station_name, + product="RAW", + base_dir=None, +): + """Create the /metadata, /issue and /data/ directories of a station.""" + # Create directory structure + _ = create_station_directory( + base_dir=base_dir, + product=product, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + _ = create_metadata_directory( + base_dir=base_dir, product=product, data_source=data_source, campaign_name=campaign_name + ) + + if product.upper() == "RAW": + _ = create_issue_directory(base_dir=base_dir, data_source=data_source, campaign_name=campaign_name) + + +def create_initial_station_structure( + data_source, + campaign_name, + station_name, + base_dir=None, +): + """Create the DISDRODB Data Archive structure for a single station.""" + from disdrodb.issue.writer import create_station_issue + from disdrodb.metadata.writer import create_station_metadata + + # Check if already been defined + # - Check presence of metadata file + metadata_filepath = define_metadata_filepath( + base_dir=base_dir, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + product="RAW", + check_exists=False, + ) + if os.path.exists(metadata_filepath): + raise ValueError( + f"A metadata file already exists at {metadata_filepath}. " + "The station is already part of the DISDRODB Archive or " + "or you already initialized the directory structure for the station !" + ) + + # Create directory structure (/metadata, /issue and /data/) + _create_station_directories( + base_dir=base_dir, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + product="RAW", + ) + + # Add default station metadata file + create_station_metadata( + base_dir=base_dir, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + product="RAW", + ) + # Add default station issue file + create_station_issue( + base_dir=base_dir, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + + # Report location of the campaign directory + campaign_dir = define_campaign_dir( + base_dir=base_dir, data_source=data_source, campaign_name=campaign_name, product="RAW" + ) + print(f"Initial station directory structure created at: {campaign_dir}") + + +#### Testing upload/download +def create_test_archive(test_base_dir, data_source, campaign_name, station_name, base_dir=None, force=False): """Create test DISDRODB Archive for a single station.""" # Check test_base_dir is not equal to true base_dir if test_base_dir == get_base_dir(): raise ValueError("Provide a test_base_dir directory different from the true DISDRODB base directory !") + # Create test DISDRODB base directory remove_if_exists(test_base_dir, force=force) os.makedirs(test_base_dir, exist_ok=True) - # Create directory structure - _ = create_station_directory( + + # Create directories (/metadata, /issue and /data/) + _create_station_directories( base_dir=test_base_dir, product="RAW", data_source=data_source, campaign_name=campaign_name, station_name=station_name, ) - _ = create_metadata_directory( - base_dir=test_base_dir, product="RAW", data_source=data_source, campaign_name=campaign_name - ) - _ = create_issue_directory(base_dir=test_base_dir, data_source=data_source, campaign_name=campaign_name) - + # Copy metadata and issue files in the test archive src_metadata_fpath = define_metadata_filepath( - product="RAW", data_source=data_source, campaign_name=campaign_name, station_name=station_name + product="RAW", + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + base_dir=base_dir, ) dst_metadata_fpath = define_metadata_filepath( product="RAW", @@ -327,7 +417,10 @@ def create_test_archive(test_base_dir, data_source, campaign_name, station_name, base_dir=test_base_dir, ) src_issue_fpath = define_issue_filepath( - data_source=data_source, campaign_name=campaign_name, station_name=station_name + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + base_dir=base_dir, ) dst_issue_fpath = define_issue_filepath( data_source=data_source, campaign_name=campaign_name, station_name=station_name, base_dir=test_base_dir diff --git a/disdrodb/api/scripts/disdrodb_initialize_station.py b/disdrodb/api/scripts/disdrodb_initialize_station.py new file mode 100644 index 00000000..a35d5f43 --- /dev/null +++ b/disdrodb/api/scripts/disdrodb_initialize_station.py @@ -0,0 +1,67 @@ +# -----------------------------------------------------------------------------. +# Copyright (c) 2021-2023 DISDRODB developers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# -----------------------------------------------------------------------------. +import sys + +import click + +from disdrodb.utils.scripts import click_station_arguments, click_base_dir_option + +sys.tracebacklimit = 0 # avoid full traceback error if occur + +# -------------------------------------------------------------------------. +# Click Command Line Interface decorator + + +@click.command() +@click_station_arguments +@click_base_dir_option +def disdrodb_initialize_station( + # Station arguments + data_source: str, + campaign_name: str, + station_name: str, + # Processing options + base_dir: str = None, +): + """Initialize the DISDRODB directory structure for a station. + + It adds the relevant directories and the default issue and metadata YAML files.. + + Parameters \n + ---------- \n + data_source : str \n + Institution name (when campaign data spans more than 1 country), or country (when all campaigns (or sensor + networks) are inside a given country).\n + Must be UPPER CASE.\n + campaign_name : str \n + Campaign name. Must be UPPER CASE.\n + station_name : str \n + Station name \n + base_dir : str \n + Base directory of DISDRODB \n + Format: <...>/DISDRODB \n + If not specified, uses path specified in the DISDRODB active configuration. \n + """ + from disdrodb.l0.routines import create_initial_station_structure + + create_initial_station_structure( + base_dir=base_dir, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + return None diff --git a/disdrodb/data_transfer/download_data.py b/disdrodb/data_transfer/download_data.py index c3c53bd9..94a0f6a4 100644 --- a/disdrodb/data_transfer/download_data.py +++ b/disdrodb/data_transfer/download_data.py @@ -89,16 +89,10 @@ def click_download_options(function: object): "--force", type=bool, show_default=True, - default=True, + default=False, help="Force overwriting", )(function) - function = click.option( - "--base_dir", - type=str, - show_default=True, - default=None, - help="DISDRODB base directory", - )(function) + return function diff --git a/disdrodb/data_transfer/scripts/disdrodb_download_archive.py b/disdrodb/data_transfer/scripts/disdrodb_download_archive.py index deef6dbe..3cfa5452 100644 --- a/disdrodb/data_transfer/scripts/disdrodb_download_archive.py +++ b/disdrodb/data_transfer/scripts/disdrodb_download_archive.py @@ -21,19 +21,21 @@ import click from disdrodb.data_transfer.download_data import click_download_archive_options, click_download_options +from disdrodb.utils.scripts import click_base_dir_option sys.tracebacklimit = 0 # avoid full traceback error if occur @click.command() @click_download_archive_options +@click_base_dir_option @click_download_options def disdrodb_download_archive( - data_sources=None, - campaign_names=None, - station_names=None, - base_dir=None, - force=True, + data_sources: str = None, + campaign_names: str = None, + station_names: str = None, + base_dir: str = None, + force: bool = False, ): from disdrodb.data_transfer.download_data import download_archive from disdrodb.utils.scripts import parse_arg_to_list diff --git a/disdrodb/data_transfer/scripts/disdrodb_download_station.py b/disdrodb/data_transfer/scripts/disdrodb_download_station.py index 2dedcba4..be840595 100644 --- a/disdrodb/data_transfer/scripts/disdrodb_download_station.py +++ b/disdrodb/data_transfer/scripts/disdrodb_download_station.py @@ -21,21 +21,23 @@ import sys import click -from disdrodb.utils.scripts import click_station_arguments + from disdrodb.data_transfer.download_data import click_download_options +from disdrodb.utils.scripts import click_base_dir_option, click_station_arguments sys.tracebacklimit = 0 # avoid full traceback error if occur @click.command() @click_station_arguments +@click_base_dir_option @click_download_options def disdrodb_download_station( - data_source, - campaign_name, - station_name, - base_dir=None, - force=False, + data_source: str, + campaign_name: str, + station_name: str, + base_dir: str = None, + force: bool = False, ): from disdrodb.data_transfer.download_data import download_station @@ -45,4 +47,4 @@ def disdrodb_download_station( campaign_name=campaign_name, station_name=station_name, force=force, - ) \ No newline at end of file + ) diff --git a/disdrodb/data_transfer/scripts/disdrodb_upload_archive.py b/disdrodb/data_transfer/scripts/disdrodb_upload_archive.py index daa6b94d..f04a28e6 100644 --- a/disdrodb/data_transfer/scripts/disdrodb_upload_archive.py +++ b/disdrodb/data_transfer/scripts/disdrodb_upload_archive.py @@ -23,6 +23,7 @@ import click from disdrodb.data_transfer.upload_data import click_upload_archive_options, click_upload_options +from disdrodb.utils.scripts import click_base_dir_option sys.tracebacklimit = 0 # avoid full traceback error if occur @@ -30,13 +31,14 @@ @click.command() @click_upload_archive_options @click_upload_options +@click_base_dir_option def disdrodb_upload_archive( - base_dir=None, - data_sources=None, - campaign_names=None, - station_names=None, - platform=None, - force=False, + base_dir: str = None, + data_sources: str = None, + campaign_names: str = None, + station_names: str = None, + platform: str = None, + force: bool = False, ): from disdrodb.data_transfer.upload_data import upload_archive from disdrodb.utils.scripts import parse_arg_to_list diff --git a/disdrodb/data_transfer/scripts/disdrodb_upload_station.py b/disdrodb/data_transfer/scripts/disdrodb_upload_station.py index 6f9a61b9..878c7884 100644 --- a/disdrodb/data_transfer/scripts/disdrodb_upload_station.py +++ b/disdrodb/data_transfer/scripts/disdrodb_upload_station.py @@ -22,7 +22,7 @@ import click -from disdrodb.data_transfer.upload_data import click_upload_options +from disdrodb.data_transfer.upload_data import click_base_dir_option, click_upload_options from disdrodb.utils.scripts import click_station_arguments sys.tracebacklimit = 0 # avoid full traceback error if occur @@ -31,13 +31,14 @@ @click.command() @click_station_arguments @click_upload_options +@click_base_dir_option def disdrodb_upload_station( - data_source, - campaign_name, - station_name, - platform=None, - base_dir=None, - force=False, + data_source: str, + campaign_name: str, + station_name: str, + platform: str = None, + base_dir: str = None, + force: bool = False, ): from disdrodb.data_transfer.upload_data import upload_station diff --git a/disdrodb/data_transfer/upload_data.py b/disdrodb/data_transfer/upload_data.py index 9600ae32..f3e63328 100644 --- a/disdrodb/data_transfer/upload_data.py +++ b/disdrodb/data_transfer/upload_data.py @@ -41,16 +41,9 @@ def click_upload_options(function: object): "--force", type=bool, show_default=True, - default=True, + default=False, help="Force uploading even if data already exists on another remote location.", )(function) - function = click.option( - "--base_dir", - type=str, - show_default=True, - default=None, - help="DISDRODB base directory", - )(function) return function @@ -94,7 +87,7 @@ def click_upload_archive_options(function: object): return function -def _check_if_upload(metadata_filepath, force): +def _check_if_upload(metadata_filepath: str, force: bool): """Check if data must be uploaded.""" if not force: disdrodb_data_url = read_yaml(metadata_filepath).get("disdrodb_data_url", "") diff --git a/disdrodb/issue/writer.py b/disdrodb/issue/writer.py index 45b30884..5d9f24c8 100644 --- a/disdrodb/issue/writer.py +++ b/disdrodb/issue/writer.py @@ -19,9 +19,12 @@ """Issue YAML File Writer.""" import logging +import os import yaml +from disdrodb.api.path import define_issue_filepath + logger = logging.getLogger(__name__) @@ -116,3 +119,42 @@ def write_default_issue(filepath: str) -> None: """ _write_issue(filepath=filepath) return None + + +def create_station_issue(data_source, campaign_name, station_name, base_dir=None): + """Write an empty YAML issue YAML file for a DISDRODB station. + + An error is raised if the file already exists ! + + Parameters + ---------- + data_source : str + The name of the institution (for campaigns spanning multiple countries) or + the name of the country (for campaigns or sensor networks within a single country). + Must be provided in UPPER CASE. + campaign_name : str + The name of the campaign. Must be provided in UPPER CASE. + station_name : str + The name of the station. + base_dir : str, optional + The base directory of DISDRODB, expected in the format ``<...>/DISDRODB``. + If not specified, the path specified in the DISDRODB active configuration will be used. + + """ + # Define issue filepath + issue_filepath = define_issue_filepath( + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + base_dir=base_dir, + check_exists=False, + ) + if os.path.exists(issue_filepath): + raise ValueError("A issue YAML file already exists at {issue_filepath}.") + # Create issue dir if not existing + issue_dir = os.path.dirname(issue_filepath) + os.makedirs(issue_dir, exist_ok=True) + # Write issue file + write_default_issue(filepath=issue_filepath) + print(f"An empty issue YAML file for station {station_name} has been created .") + return None diff --git a/disdrodb/l0/routines.py b/disdrodb/l0/routines.py index cd29be0f..fa35bcea 100644 --- a/disdrodb/l0/routines.py +++ b/disdrodb/l0/routines.py @@ -104,13 +104,6 @@ def click_l0_processing_options(function: object): default=False, help="Force overwriting", )(function) - function = click.option( - "--base_dir", - type=str, - show_default=True, - default=None, - help="DISDRODB base directory", - )(function) return function @@ -177,13 +170,6 @@ def click_l0b_concat_options(function: object): default=False, help="If true, remove all source L0B files once L0B concatenation is terminated.", )(function) - function = click.option( - "--base_dir", - type=str, - show_default=True, - default=None, - help="DISDRODB base directory", - )(function) function = click.option("-v", "--verbose", type=bool, show_default=True, default=False, help="Verbose")(function) return function diff --git a/disdrodb/l0/scripts/disdrodb_run_l0.py b/disdrodb/l0/scripts/disdrodb_run_l0.py index 7af02949..c8fee3f3 100644 --- a/disdrodb/l0/scripts/disdrodb_run_l0.py +++ b/disdrodb/l0/scripts/disdrodb_run_l0.py @@ -23,7 +23,7 @@ click_l0_processing_options, click_l0_stations_options, ) -from disdrodb.utils.scripts import parse_arg_to_list +from disdrodb.utils.scripts import click_base_dir_option, parse_arg_to_list sys.tracebacklimit = 0 # avoid full traceback error if occur @@ -32,11 +32,12 @@ @click_l0_stations_options @click_l0_archive_options @click_l0_processing_options +@click_base_dir_option def disdrodb_run_l0( # L0 disdrodb stations options - data_sources=None, - campaign_names=None, - station_names=None, + data_sources: str = None, + campaign_names: str = None, + station_names: str = None, # L0 archive options l0a_processing: bool = True, l0b_processing: bool = True, diff --git a/disdrodb/l0/scripts/disdrodb_run_l0_station.py b/disdrodb/l0/scripts/disdrodb_run_l0_station.py index 858c13b7..8bd83894 100644 --- a/disdrodb/l0/scripts/disdrodb_run_l0_station.py +++ b/disdrodb/l0/scripts/disdrodb_run_l0_station.py @@ -22,7 +22,7 @@ click_l0_archive_options, click_l0_processing_options, ) -from disdrodb.utils.scripts import click_station_arguments +from disdrodb.utils.scripts import click_base_dir_option, click_station_arguments sys.tracebacklimit = 0 # avoid full traceback error if occur @@ -34,11 +34,12 @@ @click_station_arguments @click_l0_processing_options @click_l0_archive_options +@click_base_dir_option def disdrodb_run_l0_station( # Station arguments - data_source, - campaign_name, - station_name, + data_source: str, + campaign_name: str, + station_name: str, # L0 archive options l0a_processing: bool = True, l0b_processing: bool = True, diff --git a/disdrodb/l0/scripts/disdrodb_run_l0a.py b/disdrodb/l0/scripts/disdrodb_run_l0a.py index 4d3e1537..9f924333 100644 --- a/disdrodb/l0/scripts/disdrodb_run_l0a.py +++ b/disdrodb/l0/scripts/disdrodb_run_l0a.py @@ -23,7 +23,7 @@ click_l0_processing_options, click_l0_stations_options, ) -from disdrodb.utils.scripts import parse_arg_to_list +from disdrodb.utils.scripts import click_base_dir_option, parse_arg_to_list sys.tracebacklimit = 0 # avoid full traceback error if occur @@ -31,11 +31,12 @@ @click.command() @click_l0_stations_options @click_l0_processing_options +@click_base_dir_option def disdrodb_run_l0a( # L0 disdrodb stations options - data_sources=None, - campaign_names=None, - station_names=None, + data_sources: str = None, + campaign_names: str = None, + station_names: str = None, # Processing options force: bool = False, verbose: bool = True, diff --git a/disdrodb/l0/scripts/disdrodb_run_l0a_station.py b/disdrodb/l0/scripts/disdrodb_run_l0a_station.py index f77eb96d..0faac0fd 100644 --- a/disdrodb/l0/scripts/disdrodb_run_l0a_station.py +++ b/disdrodb/l0/scripts/disdrodb_run_l0a_station.py @@ -19,7 +19,7 @@ import click from disdrodb.l0.routines import click_l0_processing_options -from disdrodb.utils.scripts import click_station_arguments +from disdrodb.utils.scripts import click_base_dir_option, click_station_arguments sys.tracebacklimit = 0 # avoid full traceback error if occur @@ -30,11 +30,12 @@ @click.command() @click_station_arguments @click_l0_processing_options +@click_base_dir_option def disdrodb_run_l0a_station( # Station arguments - data_source, - campaign_name, - station_name, + data_source: str, + campaign_name: str, + station_name: str, # Processing options force: bool = False, verbose: bool = False, diff --git a/disdrodb/l0/scripts/disdrodb_run_l0b.py b/disdrodb/l0/scripts/disdrodb_run_l0b.py index 78163cd1..518fa5bd 100644 --- a/disdrodb/l0/scripts/disdrodb_run_l0b.py +++ b/disdrodb/l0/scripts/disdrodb_run_l0b.py @@ -23,7 +23,7 @@ click_l0_processing_options, click_l0_stations_options, ) -from disdrodb.utils.scripts import parse_arg_to_list +from disdrodb.utils.scripts import click_base_dir_option, parse_arg_to_list sys.tracebacklimit = 0 # avoid full traceback error if occur @@ -31,11 +31,12 @@ @click.command() @click_l0_stations_options @click_l0_processing_options +@click_base_dir_option def disdrodb_run_l0b( # L0 disdrodb stations options - data_sources=None, - campaign_names=None, - station_names=None, + data_sources: str = None, + campaign_names: str = None, + station_names: str = None, # Processing options force: bool = False, verbose: bool = True, diff --git a/disdrodb/l0/scripts/disdrodb_run_l0b_concat.py b/disdrodb/l0/scripts/disdrodb_run_l0b_concat.py index 4852b48a..03088a62 100644 --- a/disdrodb/l0/scripts/disdrodb_run_l0b_concat.py +++ b/disdrodb/l0/scripts/disdrodb_run_l0b_concat.py @@ -25,7 +25,7 @@ click_l0_stations_options, click_l0b_concat_options, ) -from disdrodb.utils.scripts import parse_arg_to_list +from disdrodb.utils.scripts import click_base_dir_option, parse_arg_to_list sys.tracebacklimit = 0 # avoid full traceback error if occur @@ -33,8 +33,14 @@ @click.command() @click_l0_stations_options @click_l0b_concat_options +@click_base_dir_option def disdrodb_run_l0b_concat( - data_sources, campaign_names, station_names, remove_l0b=False, verbose=True, base_dir: str = None + data_sources: str = None, + campaign_names: str = None, + station_names: str = None, + remove_l0b: bool = False, + verbose: bool = True, + base_dir: str = None, ): """Run the L0B concatenation of available DISDRODB stations. diff --git a/disdrodb/l0/scripts/disdrodb_run_l0b_concat_station.py b/disdrodb/l0/scripts/disdrodb_run_l0b_concat_station.py index da587481..189343b8 100644 --- a/disdrodb/l0/scripts/disdrodb_run_l0b_concat_station.py +++ b/disdrodb/l0/scripts/disdrodb_run_l0b_concat_station.py @@ -22,7 +22,7 @@ import click from disdrodb.l0.routines import click_l0b_concat_options -from disdrodb.utils.scripts import click_station_arguments +from disdrodb.utils.scripts import click_base_dir_option, click_station_arguments sys.tracebacklimit = 0 # avoid full traceback error if occur @@ -30,11 +30,12 @@ @click.command() @click_station_arguments @click_l0b_concat_options +@click_base_dir_option def disdrodb_run_l0b_concat_station( # Station arguments - data_source, - campaign_name, - station_name, + data_source: str, + campaign_name: str, + station_name: str, # L0B concat options remove_l0b=False, verbose=True, diff --git a/disdrodb/l0/scripts/disdrodb_run_l0b_station.py b/disdrodb/l0/scripts/disdrodb_run_l0b_station.py index c8b78f86..a2fb0eb0 100644 --- a/disdrodb/l0/scripts/disdrodb_run_l0b_station.py +++ b/disdrodb/l0/scripts/disdrodb_run_l0b_station.py @@ -19,7 +19,7 @@ import click from disdrodb.l0.routines import click_l0_processing_options -from disdrodb.utils.scripts import click_station_arguments +from disdrodb.utils.scripts import click_base_dir_option, click_station_arguments sys.tracebacklimit = 0 # avoid full traceback error if occur @@ -30,11 +30,12 @@ @click.command() @click_station_arguments @click_l0_processing_options +@click_base_dir_option def disdrodb_run_l0b_station( # Station arguments - data_source, - campaign_name, - station_name, + data_source: str, + campaign_name: str, + station_name: str, # Processing options force: bool = False, verbose: bool = True, diff --git a/disdrodb/metadata/__init__.py b/disdrodb/metadata/__init__.py index dd1d097f..a07d44f0 100644 --- a/disdrodb/metadata/__init__.py +++ b/disdrodb/metadata/__init__.py @@ -1,4 +1,5 @@ from disdrodb.metadata.info import get_archive_metadata_key_value -from disdrodb.metadata.io import get_list_metadata, read_station_metadata +from disdrodb.metadata.reader import read_station_metadata +from disdrodb.metadata.search import get_list_metadata __all__ = ["read_station_metadata", "get_list_metadata", "get_archive_metadata_key_value"] diff --git a/disdrodb/metadata/check_metadata.py b/disdrodb/metadata/check_metadata.py index 915b1078..70d54c95 100644 --- a/disdrodb/metadata/check_metadata.py +++ b/disdrodb/metadata/check_metadata.py @@ -28,7 +28,8 @@ infer_data_source_from_path, ) from disdrodb.configs import get_base_dir -from disdrodb.metadata.io import get_list_metadata, read_station_metadata +from disdrodb.metadata.reader import read_station_metadata +from disdrodb.metadata.search import get_list_metadata from disdrodb.metadata.standards import get_valid_metadata_keys from disdrodb.utils.yaml import read_yaml diff --git a/disdrodb/metadata/info.py b/disdrodb/metadata/info.py index 85581c2d..b3e4c89f 100644 --- a/disdrodb/metadata/info.py +++ b/disdrodb/metadata/info.py @@ -24,7 +24,8 @@ infer_data_source_from_path, ) from disdrodb.configs import get_base_dir -from disdrodb.metadata.io import get_list_metadata, read_station_metadata +from disdrodb.metadata.reader import read_station_metadata +from disdrodb.metadata.search import get_list_metadata def get_archive_metadata_key_value(key: str, return_tuple: bool = True, base_dir: str = None): diff --git a/disdrodb/metadata/reader.py b/disdrodb/metadata/reader.py new file mode 100644 index 00000000..2ce901ff --- /dev/null +++ b/disdrodb/metadata/reader.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 + +# -----------------------------------------------------------------------------. +# Copyright (c) 2021-2023 DISDRODB developers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# -----------------------------------------------------------------------------. +"""Routines to read the DISDRODB Metadata.""" + + +from disdrodb.api.path import define_metadata_filepath +from disdrodb.utils.yaml import read_yaml + + +def read_station_metadata(data_source, campaign_name, station_name, base_dir=None, product="RAW"): + """Open the station metadata YAML file into a dictionary. + + Parameters + ---------- + data_source : str + The name of the institution (for campaigns spanning multiple countries) or + the name of the country (for campaigns or sensor networks within a single country). + Must be provided in UPPER CASE. + campaign_name : str + The name of the campaign. Must be provided in UPPER CASE. + station_name : str + The name of the station. + base_dir : str, optional + The base directory of DISDRODB, expected in the format ``<...>/DISDRODB``. + If not specified, the path specified in the DISDRODB active configuration will be used. + product : str, optional + The DISDRODB product in which to search for the metadata file. + The default is "RAW". + + Returns + ------- + metadata: dictionary + The station metadata dictionary + + """ + # Retrieve metadata filepath + metadata_filepath = define_metadata_filepath( + base_dir=base_dir, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + product=product, + check_exists=True, + ) + metadata_dict = read_yaml(metadata_filepath) + return metadata_dict diff --git a/disdrodb/metadata/scripts/disdrodb_check_metadata_archive.py b/disdrodb/metadata/scripts/disdrodb_check_metadata_archive.py index cc844fb2..80d990f5 100644 --- a/disdrodb/metadata/scripts/disdrodb_check_metadata_archive.py +++ b/disdrodb/metadata/scripts/disdrodb_check_metadata_archive.py @@ -19,11 +19,13 @@ import click +from disdrodb.utils.scripts import click_base_dir_option + sys.tracebacklimit = 0 # avoid full traceback error if occur @click.command() -@click.option("--base_dir", type=str, show_default=True, default=None, help="DISDRODB base directory") +@click_base_dir_option @click.option( "--raise_error", type=bool, show_default=True, default=True, help="Whether to raise error of finish the check" ) diff --git a/disdrodb/metadata/io.py b/disdrodb/metadata/search.py similarity index 60% rename from disdrodb/metadata/io.py rename to disdrodb/metadata/search.py index 63b7145e..8c29ffad 100644 --- a/disdrodb/metadata/io.py +++ b/disdrodb/metadata/search.py @@ -29,44 +29,6 @@ from disdrodb.utils.yaml import read_yaml, write_yaml -def read_station_metadata(data_source, campaign_name, station_name, base_dir=None, product="RAW"): - """Open the station metadata YAML file into a dictionary. - - Parameters - ---------- - data_source : str - The name of the institution (for campaigns spanning multiple countries) or - the name of the country (for campaigns or sensor networks within a single country). - Must be provided in UPPER CASE. - campaign_name : str - The name of the campaign. Must be provided in UPPER CASE. - station_name : str - The name of the station. - base_dir : str, optional - The base directory of DISDRODB, expected in the format ``<...>/DISDRODB``. - If not specified, the path specified in the DISDRODB active configuration will be used. - product : str, optional - The DISDRODB product in which to search for the metadata file. - The default is "RAW". - - Returns - ------- - metadata: dictionary - The station metadata dictionary - - """ - # Retrieve metadata filepath - metadata_filepath = define_metadata_filepath( - base_dir=base_dir, - data_source=data_source, - campaign_name=campaign_name, - station_name=station_name, - product=product, - check_exists=True, - ) - metadata_dict = read_yaml(metadata_filepath) - return metadata_dict - def get_list_metadata( data_sources=None, @@ -252,98 +214,3 @@ def _get_list_metadata_with_data(base_dir, data_sources=None, campaign_names=Non ####--------------------------------------------------------------------------. #### Default (empty) metadata -def _create_default_metadata_dict() -> dict: - """Get DISDRODB metadata default values. - - Returns - ------- - dict - Dictionary of attributes standard - """ - # Get valid metadata keys - list_attrs = get_valid_metadata_keys() - attrs = {key: "" for key in list_attrs} - - # Add default values for certain keys - attrs["latitude"] = -9999 - attrs["longitude"] = -9999 - attrs["altitude"] = -9999 - attrs["raw_data_format"] = "txt" # ['txt', 'netcdf'] - attrs["platform_type"] = "fixed" # ['fixed', 'mobile'] - return attrs - - -def write_default_metadata(filepath: str) -> None: - """Create default YAML metadata file at the specified filepath. - - Parameters - ---------- - filepath : str - File path - """ - # Get default metadata dict - metadata = _create_default_metadata_dict() - - # Try infer the data_source, campaign_name and station_name from filepath - try: - campaign_name = infer_campaign_name_from_path(filepath) - data_source = infer_data_source_from_path(filepath) - station_name = os.path.basename(filepath).split(".yml")[0] - metadata["data_source"] = data_source - metadata["campaign_name"] = campaign_name - metadata["station_name"] = station_name - except Exception: - pass - - # Write the metadata - metadata = sort_metadata_dictionary(metadata) - write_yaml(metadata, filepath=filepath, sort_keys=False) - return None - - -def create_station_metadata(data_source, campaign_name, station_name, base_dir=None, product="RAW"): - """Create a default (semi-empty) YAML metadata file for a DISDRODB station. - - An error is raised if the file already exists ! - - Parameters - ---------- - data_source : str - The name of the institution (for campaigns spanning multiple countries) or - the name of the country (for campaigns or sensor networks within a single country). - Must be provided in UPPER CASE. - campaign_name : str - The name of the campaign. Must be provided in UPPER CASE. - station_name : str - The name of the station. - base_dir : str, optional - The base directory of DISDRODB, expected in the format ``<...>/DISDRODB``. - If not specified, the path specified in the DISDRODB active configuration will be used. - product : str, optional - The DISDRODB product in which to search for the metadata file. - The default is "RAW". - - Returns - ------- - metadata: dictionary - The station metadata dictionary - - """ - # Define metadata filepath - metadata_filepath = define_metadata_filepath( - data_source=data_source, - campaign_name=campaign_name, - station_name=station_name, - base_dir=base_dir, - product=product, - check_exists=False, - ) - if os.path.exists(metadata_filepath): - raise ValueError("A metadata YAML file already exists at {metadata_filepath}.") - # Create metadata dir if not existing - metadata_dir = os.path.dirname(metadata_filepath) - os.makedirs(metadata_dir, exist_ok=True) - # Write metadata file - write_default_metadata(filepath=metadata_filepath) - print(f"An empty metadata for station {station_name} has been created .") - return None diff --git a/disdrodb/metadata/writer.py b/disdrodb/metadata/writer.py new file mode 100644 index 00000000..0f954108 --- /dev/null +++ b/disdrodb/metadata/writer.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python3 + +# -----------------------------------------------------------------------------. +# Copyright (c) 2021-2023 DISDRODB developers +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# -----------------------------------------------------------------------------. +"""Routines to write the DISDRODB Metadata.""" + +import os + +from disdrodb.api.info import infer_campaign_name_from_path, infer_data_source_from_path +from disdrodb.api.path import define_metadata_filepath +from disdrodb.metadata.manipulation import sort_metadata_dictionary +from disdrodb.metadata.standards import get_valid_metadata_keys +from disdrodb.utils.yaml import write_yaml + + +def _create_default_metadata_dict() -> dict: + """Get DISDRODB metadata default values. + + Returns + ------- + dict + Dictionary of attributes standard + """ + # Get valid metadata keys + list_attrs = get_valid_metadata_keys() + attrs = {key: "" for key in list_attrs} + + # Add default values for certain keys + attrs["latitude"] = -9999 + attrs["longitude"] = -9999 + attrs["altitude"] = -9999 + attrs["raw_data_format"] = "txt" # ['txt', 'netcdf'] + attrs["platform_type"] = "fixed" # ['fixed', 'mobile'] + return attrs + + +def write_default_metadata(filepath: str) -> None: + """Write a default YAML metadata file at the specified filepath. + + Parameters + ---------- + filepath : str + File path + """ + # Get default metadata dict + metadata = _create_default_metadata_dict() + + # Try infer the data_source, campaign_name and station_name from filepath + try: + campaign_name = infer_campaign_name_from_path(filepath) + data_source = infer_data_source_from_path(filepath) + station_name = os.path.basename(filepath).split(".yml")[0] + metadata["data_source"] = data_source + metadata["campaign_name"] = campaign_name + metadata["station_name"] = station_name + except Exception: + pass + + # Write the metadata + metadata = sort_metadata_dictionary(metadata) + + write_yaml(metadata, filepath=filepath, sort_keys=False) + return None + + +def create_station_metadata(data_source, campaign_name, station_name, base_dir=None, product="RAW"): + """Write a default (semi-empty) YAML metadata file for a DISDRODB station. + + An error is raised if the file already exists ! + + Parameters + ---------- + data_source : str + The name of the institution (for campaigns spanning multiple countries) or + the name of the country (for campaigns or sensor networks within a single country). + Must be provided in UPPER CASE. + campaign_name : str + The name of the campaign. Must be provided in UPPER CASE. + station_name : str + The name of the station. + base_dir : str, optional + The base directory of DISDRODB, expected in the format ``<...>/DISDRODB``. + If not specified, the path specified in the DISDRODB active configuration will be used. + product : str, optional + The DISDRODB product in which to search for the metadata file. + The default is "RAW". + + """ + # Define metadata filepath + metadata_filepath = define_metadata_filepath( + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + base_dir=base_dir, + product=product, + check_exists=False, + ) + if os.path.exists(metadata_filepath): + raise ValueError("A metadata YAML file already exists at {metadata_filepath}.") + # Create metadata dir if not existing + metadata_dir = os.path.dirname(metadata_filepath) + os.makedirs(metadata_dir, exist_ok=True) + # Write metadata file + write_default_metadata(filepath=metadata_filepath) + print(f"An empty metadata YAML file for station {station_name} has been created .") + return None diff --git a/disdrodb/tests/test_api/test_create_directories.py b/disdrodb/tests/test_api/test_create_directories.py index b4d2374a..f0f5882f 100644 --- a/disdrodb/tests/test_api/test_create_directories.py +++ b/disdrodb/tests/test_api/test_create_directories.py @@ -27,11 +27,14 @@ _copy_station_metadata, create_directory_structure, create_initial_directory_structure, + create_initial_station_structure, create_issue_directory, create_metadata_directory, + create_test_archive, ) from disdrodb.api.path import ( define_campaign_dir, + define_issue_filepath, define_metadata_dir, define_metadata_filepath, define_station_dir, @@ -263,6 +266,96 @@ def test_create_directory_structure(tmp_path, mocker): ) +def test_create_initial_station_structure(tmp_path): + """Check creation of station initial files and directories.""" + base_dir = tmp_path / "DISDRODB" + campaign_name = "CAMPAIGN_NAME" + data_source = "DATA_SOURCE" + station_name = "station_name" + # Create initial station structure + create_initial_station_structure( + base_dir=base_dir, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + # Check metadata and issue files have been created + metadata_filepath = define_metadata_filepath( + base_dir=base_dir, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + product="RAW", + ) + issue_filepath = define_issue_filepath( + base_dir=base_dir, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + assert os.path.exists(metadata_filepath) + assert os.path.exists(issue_filepath) + + # Check that if called once again, it raise error + with pytest.raises(ValueError): + create_initial_station_structure( + base_dir=base_dir, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + + +def test_create_test_archive(tmp_path): + """Check creation of test archive.""" + base_dir = tmp_path / "base" / "DISDRODB" + test_base_dir = tmp_path / "test" / "DISDRODB" + + campaign_name = "CAMPAIGN_NAME" + data_source = "DATA_SOURCE" + station_name = "station_name" + # Create initial station structure + create_initial_station_structure( + base_dir=base_dir, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + + create_test_archive( + test_base_dir=test_base_dir, + base_dir=base_dir, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + # Check metadata and issue files have been created + metadata_filepath = define_metadata_filepath( + base_dir=test_base_dir, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + product="RAW", + ) + issue_filepath = define_issue_filepath( + base_dir=test_base_dir, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + assert os.path.exists(metadata_filepath) + assert os.path.exists(issue_filepath) + + # Check that if called once again, it raise error + with pytest.raises(ValueError): + create_initial_station_structure( + base_dir=base_dir, + data_source=data_source, + campaign_name=campaign_name, + station_name=station_name, + ) + + def test_check_campaign_name_consistency(tmp_path): base_dir = tmp_path / "DISDRODB" campaign_name = "CAMPAIGN_NAME" diff --git a/disdrodb/tests/test_metadata/test_metadata_io.py b/disdrodb/tests/test_metadata/test_metadata_io.py index 82579992..b8ede609 100644 --- a/disdrodb/tests/test_metadata/test_metadata_io.py +++ b/disdrodb/tests/test_metadata/test_metadata_io.py @@ -23,13 +23,15 @@ import pytest from disdrodb.api.path import define_metadata_filepath -from disdrodb.metadata.io import ( - _create_default_metadata_dict, +from disdrodb.metadata.reader import read_station_metadata +from disdrodb.metadata.search import ( _get_list_all_metadata, _get_list_metadata_with_data, - create_station_metadata, get_list_metadata, - read_station_metadata, +) +from disdrodb.metadata.writer import ( + _create_default_metadata_dict, + create_station_metadata, write_default_metadata, ) from disdrodb.tests.conftest import ( diff --git a/disdrodb/utils/scripts.py b/disdrodb/utils/scripts.py index cbff4f24..3fc25ff9 100644 --- a/disdrodb/utils/scripts.py +++ b/disdrodb/utils/scripts.py @@ -68,3 +68,14 @@ def click_station_arguments(function: object): function = click.argument("campaign_name", metavar="")(function) function = click.argument("data_source", metavar="")(function) return function + + +def click_base_dir_option(function: object): + function = click.option( + "--base_dir", + type=str, + show_default=True, + default=None, + help="DISDRODB base directory", + )(function) + return function diff --git a/docs/source/contribute_data.rst b/docs/source/contribute_data.rst index f2e08d86..e3888d90 100644 --- a/docs/source/contribute_data.rst +++ b/docs/source/contribute_data.rst @@ -123,7 +123,7 @@ If you contribute multiple stations, just rerun the following command for each s .. code:: bash - disdrodb_initialize_station_files + disdrodb_initialize_station The DISDRODB Raw archive will have the following structure: diff --git a/pyproject.toml b/pyproject.toml index fe4d650f..49cbae1c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,8 @@ write_to = "disdrodb/_version.py" packages = ["disdrodb"] [project.scripts] +# Initialization +disdrodb_initialize_station="disdrodb.api.create_directories.scripts.disdrodb_initialize_station:disdrodb_initialize_station" # L0A disdrodb_run_l0a_station="disdrodb.l0.scripts.disdrodb_run_l0a_station:disdrodb_run_l0a_station" disdrodb_run_l0a="disdrodb.l0.scripts.disdrodb_run_l0a:disdrodb_run_l0a"