add fre dora tools

.gitignore

@@ -172,3 +172,19 @@ cython_debug/
 #  and can be added to the global gitignore or merged into this file.  For a more nuclear
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 #.idea/
+
+# Files produced by tests.
+combined-FOO.yaml
+fre/app/regrid_xy/tests/test_inputs_outputs/in-dir/
+fre/app/regrid_xy/tests/test_inputs_outputs/out-dir/
+fre/app/regrid_xy/tests/test_inputs_outputs/out-test-dir/
+fre/app/regrid_xy/tests/test_inputs_outputs/remap-dir/
+fre/app/regrid_xy/tests/test_inputs_outputs/remap-test-dir/
+fre/app/regrid_xy/tests/test_inputs_outputs/work/
+fre/make/tests/AM5_example/combined-am5.yaml
+fre/make/tests/null_example/combined-null_model.yaml
+fre/tests/fremake_out/
+fre/tests/test_files/ocean_sos_var_file/ocean_monthly_1x1deg.199301-199712.sosV2.nc
+fre/tests/test_files/outdir/
+fregrid_remap_file_288_by_180.nc
+rose-app-run.conf

fre/dora/fredora.py

@@ -0,0 +1,42 @@
+import click
+
+from .subtools import add_experiment_to_dora, get_dora_experiment_id, \
+                      publish_analysis_figures
+
+
+@click.group(help=click.style(" - access fre dora subcommands", fg=(250, 154, 90)))
+def dora_cli():
+    """Entry point to fre dora click commands."""
+    pass
+
+
+@dora_cli.command()
+@click.option("--experiment-yaml", type=str, required=True, help="Path to the experiment yaml.")
+@click.option("--dora-url", type=str, required=False, help="Dora's URL.")
+def add(experiment_yaml, dora_url):
+    """Add an experiment to dora."""
+    print(add_experiment_to_dora(experiment_yaml, dora_url))
+
+
+@dora_cli.command()
+@click.option("--experiment-yaml", type=str, required=True, help="Path to the experiment yaml.")
+@click.option("--dora-url", type=str, required=False, help="Dora's URL.")
+def get(experiment_yaml, dora_url):
+    """Gets an experiment id from dora."""
+    print(get_dora_experiment_id(experiment_yaml, dora_url))
+
+
+@dora_cli.command()
+@click.option("--name", type=str, required=True, help="Name of the analysis script.")
+@click.option("--experiment-yaml", type=str, required=True,
+              help="Path to the experiment yaml file.")
+@click.option("--figures-yaml", type=str, required=True,
+              help="Path to the yaml that contains the figure paths.")
+@click.option("--dora-url", type=str, required=False, help="Dora's URL.")
+def publish(name, experiment_yaml, figures_yaml, dora_url):
+    """Uploads the analysis figures to dora."""
+    publish_analysis_figures(name, experiment_yaml, figures_yaml, dora_url)
+
+
+if __name__ == "__main__":
+    dora_cli()

fre/dora/subtools.py

@@ -0,0 +1,167 @@
+from os import getenv
+from pathlib import Path
+
+from requests import get, post
+from yaml import safe_load
+
+
+_dora_token = getenv("DORA_TOKEN")
+_production_dora_url = "https://dora.gfdl.noaa.gov"
+
+
+def _get_request(url, params=None):
+    """Sends a get request to the input url.
+
+    Args:
+        url: String url to send the get request to.
+        params: Dictionary of data that will be passed as URL parameters.
+
+    Returns:
+        Dictionary of response body data and string response text.
+
+    Raises:
+        ValueError if the response does not return status 200.
+    """
+    response = get(url, params)
+    if response.status_code != 200:
+        print(response.text)
+        return ValueError("get from {url} failed.")
+    return response.json(), response.text
+
+
+def _post_request(url, data, auth):
+    """Post an http request to a url.
+
+    Args:
+        url: String url to post the http request to.
+        data: Dictionary of data that will be sent in the body of the request.
+        auth: String authentication username.
+
+    Returns:
+        String text from the http response.
+
+    Raises:
+        ValueError if the response does not return status 200.
+    """
+    response = post(url, json=data, auth=(auth, None))
+    if response.status_code != 200:
+        print(response.text)
+        raise ValueError(f"post to {url} with {data['expName']} failed.")
+    return response.text
+
+
+def _parse_experiment_yaml_for_dora(path):
+    """Parse the experiment yaml and return a dictionary of the data needed to add the
+       experiment to dora.
+
+    Args:
+        path: Path to the experiment yaml.
+
+    Returns:
+        Dictionary of data needed to add the experiment to dora.
+
+    Raises:
+        ValueError if the experiment owner cannot be determined.
+    """
+    with open(path) as file_:
+        yaml_ = safe_load(file_)
+
+        # Determine the username - is this a hack?
+        history_path_parts = Path(yaml_["directories"]["history_dir"]).parts
+        user = history_path_parts[2]
+        if user == "$USER":
+            user = getenv(user[1:])
+        if not user:
+            raise ValueError(f"Could not identify user {user}.")
+
+        # Expand the paths.
+        pp_path = yaml_["directories"]["pp_dir"].replace("$USER", user)
+        database_path = pp_path.replace("archive", "home").replace("pp", "db") # Nasty hack.
+        analysis_path = yaml_["directories"]["analysis_dir"].replace("$USER", user)
+
+        # Get the model type from the history directory path - is there a better way?
+        model_type = history_path_parts[3].upper() # Nasty hack.
+
+        # Get the starting and ending years and total length of the experiment.
+        start = int(yaml_["postprocess"]["settings"]["pp_start"])
+        stop = int(yaml_["postprocess"]["settings"]["pp_stop"])
+        length = stop - start + 1
+
+        return {
+            "expLength": length,
+            "expName": yaml_["name"],
+            "expType": yaml_["name"].split("_")[-1].upper(), # Nasty hack.
+            "expYear": start,
+            "modelType": model_type,
+            "owner": user,
+            "pathAnalysis": analysis_path.rstrip("/"),
+            "pathDB": database_path.rstrip("/"),
+            "pathPP": pp_path.rstrip("/"),
+            "pathXML": path.rstrip("/"),
+            "userName": user,
+        }
+
+
+def add_experiment_to_dora(experiment_yaml, dora_url=None):
+    """Adds the experiment to dora using a http request.
+
+    Args:
+        experiment_yaml: Path to the experiment yaml.
+        dora_url: String URL for dora.
+    """
+    # Parse the experiment yaml to get the data needed to add the experiment to dora.
+    data = _parse_experiment_yaml_for_dora(experiment_yaml)
+    data["token"] = _dora_token
+
+    # Add the experiment to dora.
+    url = dora_url or _production_dora_url
+    return _get_request(f"{url}/api/add", data)[1]
+
+
+def get_dora_experiment_id(experiment_yaml, dora_url=None):
+    """Gets the experiment id using a http request after parsing the experiment yaml.
+
+    Args:
+        experiment_yaml: Path to the experiment yaml.
+        dora_url: String URL for dora.
+
+    Returns:
+        Integer dora experiment id.
+
+    Raises:
+        ValueError if the unique experiment (identified by the pp directory path)
+        cannot be found.
+    """
+    # Parse the experiment yaml to get the data needed to get the experiment id from.
+    data = _parse_experiment_yaml_for_dora(experiment_yaml)
+
+    # Get the experiment id from dora.
+    url = dora_url or _production_dora_url
+    response = _get_request(f"{url}/api/search?search={data['owner']}")
+    for experiment in response[0].values():
+        if experiment["pathPP"] and experiment["pathPP"].rstrip("/") == data["pathPP"]:
+            return int(experiment["id"])
+    raise ValueError("could not find experiment with pp directory - {data['pathPP']}")
+
+
+def publish_analysis_figures(name, experiment_yaml, figures_yaml, dora_url=None):
+    """Uploads the analysis figures to dora.
+
+    Args:
+        name: String name of the analysis script.
+        experiment_yaml: Path to the experiment yaml file.
+        figures_yaml: Path to the yaml that contains the figure paths.
+        dora_url: String URL for dora.
+    """
+    # Check to make sure that the experiment was added to dora and get it id.
+    dora_id = get_dora_experiment_id(experiment_yaml)
+
+    # Parse out the list of paths from the input yaml file and upload them.
+    url = dora_url or _production_dora_url
+    url = f"{url}/api/add-png"
+    data = {"id": dora_id, "name": name}
+    with open(figures_yaml) as file_:
+        paths = safe_load(file_)["figure_paths"]
+        for path in paths:
+            data["path"] = path
+            _post_request(url, data, _dora_token)

fre/dora/tests/test_subtools.py

@@ -0,0 +1,56 @@
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from fre.dora.subtools import add_experiment_to_dora, get_dora_experiment_id, \
+                              publish_analysis_figures
+import pytest
+
+
+def _make_experiment_yaml(path, name, whitespace="  "):
+    """Creates and experiment yaml configuration file for testing.
+
+    Args:
+        path: Path to the experiment yaml file that will be created.
+        name: String name of the analysis package.
+        whitespace: Amount of whitespace each block will be indented by.
+    """
+    analysis_path = ""
+    history_path = ""
+    pp_path = ""
+    pp_start = 1980
+    pp_stop = 1981
+    with open(path, "w") as yaml_:
+        yaml_.write("directories:\n")
+        yaml_.write(f"{whitespace}analysis_dir: {analysis_path}\n")
+        yaml_.write(f"{whitespace}history_dir: {history_path}\n")
+        yaml_.write(f"{whitespace}pp_dir: {pp_path}\n")
+        yaml_.write(f"name: {name}\n")
+        yaml_.write("postprocess:\n")
+        yaml_.write(f"{whitespace}settings:\n")
+        yaml_.write(f"{2*whitespace}pp_start: {pp_start}\n")
+        yaml_.write(f"{2*whitespace}pp_start: {pp_stop}\n")
+
+
+def test_add_experiment_to_dora():
+    name = "freanalysis_clouds"
+    with TemporaryDirectory() as tmp:
+        experiment_yaml = Path(tmp) / "experiment.yaml"
+        _make_experiment_yaml(experiment_yaml, name)
+        id_ = add_experiment_to_dora(experiment_yaml, "https://dora-dev.gfdl.noaa.gov")
+
+
+def test_get_dora_experiment_id():
+    name = "freanalysis_clouds"
+    with TemporaryDirectory() as tmp:
+        experiment_yaml = Path(tmp) / "experiment.yaml"
+        _make_experiment_yaml(experiment_yaml, name)
+        id_ = get_dora_experiment_id(experiment_yaml, "https://dora-dev.gfdl.noaa.gov")
+
+
+def test_publish_analysis_figures():
+    name = "freanalysis_clouds"
+    with TemporaryDirectory() as tmp:
+        experiment_yaml = Path(tmp) / "experiment.yaml"
+        _make_experiment_yaml(experiment_yaml, name)
+        publish_analysis_figures(name, experiment_yaml, figures_yaml,
+                                 "https://dora-dev.gfdl.noaa.gov")

fre/fre.py

@@ -10,34 +10,37 @@
 #versioning... always fun...
 # turn xxxx.y into xxxx.0y
 import importlib.metadata
+
+import click
+
+from .lazy_group import LazyGroup
+
+
 version_unexpanded = importlib.metadata.version('fre-cli')
 version_unexpanded_split = version_unexpanded.split('.')
 if len(version_unexpanded_split[1]) == 1:
-    version_minor = "0" + version_unexpanded_split[1]
+    version_minor = f"0{version_unexpanded_split[1]}"
 else:
     version_minor = version_unexpanded_split[1]
-version = version_unexpanded_split[0] + '.' + version_minor
-
+version = f"{version_unexpanded_split[0]}.{version_minor}"
 
 
-
-
-import click
-from .lazy_group import LazyGroup
-
 @click.group(
     cls = LazyGroup,
-    lazy_subcommands = {"pp": ".pp.frepp.pp_cli",
-                       "catalog": ".catalog.frecatalog.catalog_cli",
-                       "list": ".list.frelist.list_cli",
-                       "check": ".check.frecheck.check_cli",
-                       "run": ".run.frerun.run_cli",
-                       "test": ".test.fretest.test_cli",
-                       "yamltools": ".yamltools.freyamltools.yamltools_cli",
-                       "make": ".make.fremake.make_cli",
-                       "app": ".app.freapp.app_cli",
-                       "cmor": ".cmor.frecmor.cmor_cli",
-                       "analysis": ".analysis.freanalysis.analysis_cli"},
+    lazy_subcommands = {
+        "pp": ".pp.frepp.pp_cli",
+        "catalog": ".catalog.frecatalog.catalog_cli",
+        "list": ".list.frelist.list_cli",
+        "check": ".check.frecheck.check_cli",
+        "run": ".run.frerun.run_cli",
+        "test": ".test.fretest.test_cli",
+        "yamltools": ".yamltools.freyamltools.yamltools_cli",
+        "make": ".make.fremake.make_cli",
+        "app": ".app.freapp.app_cli",
+        "cmor": ".cmor.frecmor.cmor_cli",
+        "analysis": ".analysis.freanalysis.analysis_cli",
+        "dora": ".dora.fredora.dora_cli",
+    },
     help = click.style(
         "'fre' is the main CLI click group that houses the other tool groups as lazy subcommands.",
         fg='cyan')
@@ -49,8 +52,10 @@
     version=version
 )
 
+
 def fre():
     ''' entry point function to subgroup functions '''
 
+
 if __name__ == '__main__':
     fre()