First prototype of a code base

.github/workflows/ci.yaml

@@ -34,29 +34,31 @@ jobs:
     steps:
     - uses: actions/checkout@v4
 
-    - name: Set up Python 3.11
+    - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
-        python-version: 3.11
+        python-version: ${{ matrix.python-version }}
     - name: Install hatch
       run: |
         pip install --upgrade pip
         pip install hatch
+        pip install . # to get verdi
     - name: Run test suite
       env:
         PYTEST_ADDOPTS: "--durations=0"
       run: |
+        verdi presto
         hatch test --cover
 
   docs:
     runs-on: ubuntu-latest
     timeout-minutes: 15
     steps:
     - uses: actions/checkout@v4
-    - name: Set up Python 3.11
+    - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v5
       with:
-        python-version: "3.11"
+        python-version: ${{ matrix.python-version }}
     - name: Install hatch
       run: |
         pip install --upgrade pip

docs/src/api.rst

@@ -0,0 +1,20 @@
+API
+===
+
+wcflow.wc
+---------
+.. automodule:: wcflow.wc
+   :members:
+   :undoc-members:
+
+wcflow.core
+-----------
+.. automodule:: wcflow.core
+   :members: 
+   :undoc-members:
+
+wcflow.workgraph
+----------------
+.. automodule:: wcflow.workgraph
+   :members:
+   :undoc-members:

docs/src/conf.py

@@ -29,6 +29,7 @@
     'sphinx.ext.mathjax',
     'sphinx.ext.intersphinx',
     'sphinx.ext.viewcode',
+    'sphinx.ext.autodoc',
 ]
 
 # Add any paths that contain templates here, relative to this directory.

docs/src/getting_started.rst

@@ -4,3 +4,9 @@ Getting started
 .. include:: ../../README.rst
    :start-after: marker-installation
    :end-before: marker-developer-tools
+
+Example
+-------
+
+.. include:: ../../examples/README.rst
+   :start-after: marker-example-text

docs/src/index.rst

@@ -5,3 +5,4 @@
 .. toctree::
 
    getting_started.rst
+   api.rst

examples/README.rst

@@ -0,0 +1,12 @@
+Running examples
+================
+
+.. marker-example-text
+
+To run the example you can use the command
+
+.. code-block:: bash
+
+    wcflow files/config/example_config.yaml
+
+It is important to execute it from the examples folder as the path are hardcoded.

examples/files/configs/example_config.yaml

@@ -0,0 +1,181 @@
+start_date: '2026-01-01T00:00'
+end_date: '2027-01-01T00:00'
+cycles:
+  init:
+    tasks:
+      extpar:
+        inputs:
+          - obs_data:
+              arg_option: "--input"
+        outputs:
+          - extpar_file
+  icon_bimonthly:
+    period: 'P2M'
+    tasks:
+      preproc:
+        inputs:
+          - grid_file:
+              arg_option: "-g"
+          - extpar_file:
+              date: '2026-01-01T00:00'
+              arg_option: "-p"
+          - ERA5:
+              arg_option: "-e"
+        outputs:
+          - icon_input
+        depends:
+          - icon:
+              lag: '-P4M'
+      icon:
+        inputs:
+          - grid_file:
+              arg_option: "-g"
+          - icon_input:
+              arg_option: "--input"
+          #- icon_restart:
+          #    lag: '-P2M'
+        outputs:
+          - stream_1
+          - stream_2
+          #- icon_restart
+      postproc_1:
+        inputs:
+          - stream_1:
+              arg_option: "--input"
+        outputs:
+          - postout_1
+      store_and_clean_1:
+        inputs:
+          - postout_1:
+              arg_option: "--input"
+          - stream_1:
+              arg_option: "--stream"
+          - icon_input:
+              arg_option: "--icon_input"
+        outputs:
+          - stored_data_1
+  yearly:
+    period: 'P1Y'
+    tasks:
+      postproc_2:
+        inputs:
+          - stream_2:
+             lag: 
+               - 'P0M'
+             arg_option: "--input"
+        outputs:
+          - postout_2
+      store_and_clean_2:
+        inputs:
+          - postout_2:
+             arg_option: "--input"
+          - stream_2:
+             lag: 
+               - 'P0M'
+        outputs:
+          - stored_data_2
+# Each task and piece of data (input and output of tasks) used to
+# define the graph is described in that section
+tasks:
+  root:
+    # All tasks inherit the root task properties
+    host: santis
+    account: g110
+  extpar:
+    plugin: extpar
+    command: $PWD/files/scripts/extpar
+    command_option: "--verbose" # todo implement support
+    config: path/to/namelists/dir
+    uenv:
+      squashfs: path/to/squashfs
+      mount_point: runtime/mount/point
+    nodes: 1
+    walltime: '00:02:00'
+  preproc:
+    plugin: AiiDA Shell
+    command: $PWD/files/scripts/cleanup.sh
+    nodes: 4
+    walltime: '00:02:00'
+    config: path/to/config/dir
+    uenv:
+      squashfs: path/to/squashfs
+      mount_point: runtime/mount/point
+  icon:
+    plugin: icon
+    command: $PWD/files/scripts/icon
+    nodes: 40
+    walltime: '23:59:59'
+    config: path/to/namelists/dir
+    uenv:
+      squashfs: path/to/squashfs
+      mount_point: runtime/mount/point
+  postproc_1:
+    plugin: AiiDA Shell
+    command: $PWD/files/scripts/main_script_ocn.sh
+    nodes: 2
+    walltime: '00:05:00'
+    conda_env: path/to/yaml/env/file
+    uenv:
+      squashfs: path/to/squashfs
+      mount_point: runtime/mount/point
+  postproc_2:
+    plugin: AiiDA Shell
+    command: $PWD/files/scripts/main_script_atm.sh
+    nodes: 2
+    walltime: '00:05:00'
+    src: path/to/src/dir
+    conda_env: path/to/yaml/env/file
+    uenv:
+      squashfs: path/to/squashfs
+      mount_point: runtime/mount/point
+  store_and_clean_1:
+    plugin: AiiDA Shell
+    command: $PWD/files/scripts/post_clean.sh
+    nodes: 1
+    walltime: '00:01:00'
+  store_and_clean_2:
+    plugin: AiiDA Shell
+    command: $PWD/files/scripts/post_clean.sh
+    nodes: 1
+    walltime: '00:01:00'
+data:
+  preproc_output:
+    type: file
+    src: $PWD/files/data/file.ncdf
+  grid_file:
+    type: file
+    src: $PWD/files/data/grid
+  obs_data:
+    type: file
+    src: $PWD/files/data/obs_data
+  ERA5:
+    type: file
+    src: $PWD/files/data/era5
+  extpar_file:
+    type: file
+    src: output
+  icon_input:
+    type: file
+    src: output
+  icon_restart:
+    type: file
+    format: ncdf
+    src: restart
+  stream_1:
+    type: file
+    src: output_1
+  stream_2:
+    type: file
+    src: output_2
+  postout_1:
+    type: file
+    src: postout
+  postout_2:
+    type: file
+    src: postout
+  stored_data_1:
+    type: file
+    src: stored_data
+  stored_data_2:
+    type: file
+    src: stored_data

examples/files/scripts/cleanup.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+
+def main():
+    # Main script execution continues here
+    print("Cleaning")
+
+
+if __name__ == '__main__':
+    main()

examples/files/scripts/cleanup.sh

@@ -0,0 +1 @@
+echo "cleanup" > output

examples/files/scripts/extpar

@@ -0,0 +1 @@
+echo "extpar" > output

examples/files/scripts/icon

@@ -0,0 +1,4 @@
+echo "icon" > restart
+echo "icon" > output
+echo "icon" > output_1
+echo "icon" > output_2

examples/files/scripts/icon.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+
+import argparse
+from pathlib import Path
+
+def main():
+    parser = argparse.ArgumentParser(description='A simple script with an optional restart argument.')
+    parser.add_argument('icon_input', type=str, help='The icon input.')
+    parser.add_argument('--restart', nargs='?', type=str, help='The icon restart file.')
+    #parser.add_argument('--restart', nargs='?', const='default', type=str, help='Initiate a restart operation with an optional string argument.')
+
+
+    args = parser.parse_args()
+
+    output = Path('output')
+    output.write_text("")
+
+    if args.restart:
+        restart = Path(args.restart)
+        restart.read_text()
+        text = "Restart operation initiated..."
+        print(text)
+        with output.open("a") as f:
+            f.write(text)
+    else:
+        text = "No restart option provided. Continuing without restart."
+        print(text)
+        with output.open("a") as f:
+            f.write(text)
+
+    # Main script execution continues here
+    text = "Script execution continues..."
+    print(text)
+    with output.open("a") as f:
+        f.write(text)
+
+    restart = Path('restart')
+    restart.write_text("")
+
+if __name__ == '__main__':
+    main()

examples/files/scripts/icon.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+# Function to write text to the output file
+write_to_output() {
+  local text="$1"
+  echo "$text"
+  echo "$text" >> output
+}
+
+# Check if at least one argument is provided
+if [ "$#" -lt 1 ]; then
+  echo "Usage: $0 icon_input [--restart [restart_file]]"
+  exit 1
+fi
+
+# Positional argument
+icon_input="$1"
+
+# Optional restart argument
+restart_file=""
+
+if [ "$2" == "--restart" ]; then
+  if [ -n "$3" ]; then
+    restart_file="$3"
+  fi
+fi
+
+# Create/empty the output file
+> output
+
+# Handling restart if the argument is provided
+if [ -n "$restart_file" ]; then
+  if [ -f "$restart_file" ]; then
+    cat "$restart_file" > /dev/null
+    write_to_output "Restart operation initiated..."
+  else
+    echo "Restart file $restart_file does not exist."
+    exit 1
+  fi
+else
+  write_to_output "No restart option provided. Continuing without restart."
+fi
+
+# Main script execution continues here
+write_to_output "Script execution continues..."
+
+# Create/empty the restart file
+> restart

examples/files/scripts/main_script_atm.sh

@@ -0,0 +1 @@
+echo "main_script_atm.sh" > postout

examples/files/scripts/main_script_ocn.sh

@@ -0,0 +1 @@
+echo "python main_script_ocn.sh" > postout

examples/files/scripts/post_clean.sh

@@ -0,0 +1 @@
+echo "store_and_clean" > stored_data