Add routines for L1 and L2 processing

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 ---
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
@@ -12,12 +12,12 @@ repos:
       - id: check-ast
       - id: check-added-large-files
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.5
+    rev: v0.7.1
     hooks:
       - id: ruff
         args: [--fix]
   - repo: https://github.com/psf/black
-    rev: 24.3.0
+    rev: 24.10.0
     hooks:
       - id: black
         language_version: python3
@@ -27,18 +27,18 @@ repos:
       - id: blackdoc
         additional_dependencies: ["black[jupyter]"]
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: "v3.1.0"
+    rev: "v4.0.0-alpha.8"
     hooks:
       - id: prettier
         types_or: [yaml, html, css, scss, javascript, json] # markdown to avoid conflicts with mdformat
   - repo: https://github.com/codespell-project/codespell
-    rev: v2.2.6
+    rev: v2.3.0
     hooks:
       - id: codespell
         types_or: [python, markdown, rst]
         additional_dependencies: [tomli]
   - repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.2
+    rev: v3.19.0
     hooks:
       - id: pyupgrade
   - repo: https://github.com/MarcoGorelli/madforhooks
@@ -47,7 +47,7 @@ repos:
       # - id: conda-env-sorter # conflicts with prettier
       - id: check-execution-order
   - repo: https://github.com/executablebooks/mdformat
-    rev: 0.7.17
+    rev: 0.7.18
     hooks:
       - id: mdformat
         additional_dependencies: [mdformat-gfm, mdformat-black]
@@ -58,7 +58,7 @@ repos:
       - id: nbstripout
         args: [--keep-output]
   - repo: https://github.com/nbQA-dev/nbQA
-    rev: 1.8.5
+    rev: 1.8.7
     hooks:
       - id: nbqa-black
       - id: nbqa-ruff

CODE_OF_CONDUCT.md

@@ -5,7 +5,7 @@
 We as members, contributors, and leaders pledge to make participation in our
 community a harassment-free experience for everyone, regardless of age, body
 size, visible or invisible disability, ethnicity, sex characteristics, gender
-identity and expression, level of experience, education, socio-economic status,
+identity and expression, level of experience, education, socioeconomic status,
 nationality, personal appearance, race, caste, color, religion, or sexual
 identity and orientation.
 

data/DISDRODB/Raw/DATA_SOURCE/CAMPAIGN_NAME/metadata/station_name_1.yml

@@ -36,7 +36,7 @@ firmware_version: ""
 sensor_beam_length: ""
 sensor_beam_width: ""
 sensor_nominal_width: ""
-measurement_interval: ""
+measurement_interval: 30
 calibration_sensitivity: ""
 calibration_certification_date: ""
 calibration_certification_url: ""

data/DISDRODB/Raw/DATA_SOURCE/CAMPAIGN_NAME/metadata/station_name_2.yml

@@ -36,7 +36,7 @@ firmware_version: ""
 sensor_beam_length: ""
 sensor_beam_width: ""
 sensor_nominal_width: ""
-measurement_interval: ""
+measurement_interval: 30
 calibration_sensitivity: ""
 calibration_certification_date: ""
 calibration_certification_url: ""

disdrodb/__init__.py

@@ -1,4 +1,23 @@
+# -----------------------------------------------------------------------------.
+# Copyright (c) 2021-2023 DISDRODB developers
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+# -----------------------------------------------------------------------------.
+"""DISDRODB software."""
+
 import contextlib
+import importlib
 import os
 from importlib.metadata import PackageNotFoundError, version
 
@@ -18,6 +37,11 @@
     check_archive_metadata_geolocation,
 )
 
+PRODUCT_VERSION = "V0"
+SOFTWARE_VERSION = "V" + importlib.metadata.version("disdrodb")
+CONVENTIONS = "CF-1.10, ACDD-1.3"
+
+
 __all__ = [
     "define_configs",
     "available_stations",

disdrodb/api/checks.py

@@ -24,11 +24,11 @@
 
 from disdrodb.api.info import infer_disdrodb_tree_path_components
 from disdrodb.api.path import (
+    define_data_dir,
     define_issue_dir,
     define_issue_filepath,
     define_metadata_dir,
     define_metadata_filepath,
-    define_station_dir,
 )
 from disdrodb.utils.directories import (
     ensure_string_path,
@@ -70,10 +70,7 @@
         ``True`` if url well formatted, ``False`` if not well formatted.
     """
     regex = r"^(https?:\/\/)?(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_\+.~#?&//=]*)$"  # noqa: E501
-
-    if re.match(regex, url):
-        return True
-    return False
+    return re.match(regex, url)
 
 
 def check_path_is_a_directory(dir_path, path_name=""):
@@ -95,6 +92,7 @@
 def check_base_dir(base_dir: str):
     """Raise an error if the path does not end with ``DISDRODB``."""
     base_dir = str(base_dir)  # convert Pathlib to string
+    base_dir = os.path.normpath(base_dir)
     if not base_dir.endswith("DISDRODB"):
         raise ValueError(f"The path {base_dir} does not end with DISDRODB. Please check the path.")
     return base_dir
@@ -150,53 +148,76 @@
     """Check DISDRODB product."""
     if not isinstance(product, str):
         raise TypeError("`product` must be a string.")
-    valid_products = ["RAW", "L0A", "L0B"]
+    valid_products = ["RAW", "L0A", "L0B", "L0C", "L1", "L2E", "L2M", "L2S"]
     if product.upper() not in valid_products:
         msg = f"Valid `products` are {valid_products}."
         logger.error(msg)
         raise ValueError(msg)
     return product
 
 
-def check_station_dir(product, data_source, campaign_name, station_name, base_dir=None):
-    """Check existence of the station data directory. If does not exists, raise an error."""
-    station_dir = define_station_dir(
+def has_available_data(
+    data_source,
+    campaign_name,
+    station_name,
+    product,
+    base_dir=None,
+    # Option for L2E
+    sample_interval=None,
+    rolling=None,
+    # Option for L2M
+    model_name=None,
+):
+    """Return ``True`` if data are available for the given product and station."""
+    # Define product directory
+    data_dir = define_data_dir(
         product=product,
         base_dir=base_dir,
         data_source=data_source,
         campaign_name=campaign_name,
         station_name=station_name,
+        # Option for L2E
+        sample_interval=sample_interval,
+        rolling=rolling,
+        # Option for L2M
+        model_name=model_name,
+        # Directory options
         check_exists=False,
     )
-    if not os.path.exists(station_dir) and os.path.isdir(station_dir):
-        msg = f"The station {station_name} data directory does not exist at {station_dir}."
-        logger.error(msg)
-        raise ValueError(msg)
-    return station_dir
-
+    # If the product directory does not exists, return False
+    if not os.path.isdir(data_dir):
+        return False
 
-def has_available_station_files(product, data_source, campaign_name, station_name, base_dir=None):
-    """Return ``True`` if data are available for the given product and station."""
-    station_dir = check_station_dir(
-        product=product,
-        base_dir=base_dir,
-        data_source=data_source,
-        campaign_name=campaign_name,
-        station_name=station_name,
-    )
-    filepaths = list_files(station_dir, glob_pattern="*", recursive=True)
+    # If no files, return False
+    filepaths = list_files(data_dir, glob_pattern="*", recursive=True)
     nfiles = len(filepaths)
     return nfiles >= 1
 
 
-def check_station_has_data(product, data_source, campaign_name, station_name, base_dir=None):
-    """Check the station data directory has data inside. If not, raise an error."""
-    if not has_available_station_files(
+def check_data_availability(
+    product,
+    data_source,
+    campaign_name,
+    station_name,
+    base_dir=None,
+    # Option for L2E
+    sample_interval=None,
+    rolling=None,
+    # Option for L2M
+    model_name=None,
+):
+    """Check the station product data directory has files inside. If not, raise an error."""
+    if not has_available_data(
         product=product,
         base_dir=base_dir,
         data_source=data_source,
         campaign_name=campaign_name,
         station_name=station_name,
+        # Option for L2E
+        sample_interval=sample_interval,
+        rolling=rolling,
+        # Option for L2M
+        model_name=model_name,
     ):
         msg = f"The {product} station data directory of {data_source} {campaign_name} {station_name} is empty !"
         logger.error(msg)
@@ -271,6 +292,7 @@
 def check_issue_file(data_source, campaign_name, station_name, base_dir=None):
     """Check existence of a valid issue YAML file. If does not exists, raise an error."""
     from disdrodb.issue.checks import check_issue_compliance
+    from disdrodb.issue.writer import create_station_issue
 
     _ = check_issue_dir(
         base_dir=base_dir,
@@ -286,9 +308,12 @@
     )
     # Check existence
     if not os.path.exists(issue_filepath):
-        msg = f"The issue YAML file of {data_source} {campaign_name} {station_name} does not exist at {issue_filepath}."
-        logger.error(msg)
-        raise ValueError(msg)
+        create_station_issue(
+            base_dir=base_dir,
+            data_source=data_source,
+            campaign_name=campaign_name,
+            station_name=station_name,
+        )
 
     # Check validity
     check_issue_compliance(
@@ -398,7 +423,7 @@
     check_directories_inside(raw_dir)
 
     # Check there is data in the station directory
-    check_station_has_data(
+    check_data_availability(
         product="RAW",
         base_dir=base_dir,
         data_source=data_source,