From 156cb25fcef9de54d6836ce5b840c97cfb39c4c2 Mon Sep 17 00:00:00 2001 From: rem1776 Date: Fri, 1 Nov 2024 13:43:35 -0400 Subject: [PATCH 1/7] update CI image and workflow, add run-fremake test and mkmf submodule --- .github/workflows/create_test_conda_env.yml | 56 ++++++++------------- .gitmodules | 3 ++ fre/make/tests/test_fre_make_run_fremake.py | 13 +++++ mkmf | 1 + 4 files changed, 39 insertions(+), 34 deletions(-) create mode 100644 .gitmodules create mode 100644 fre/make/tests/test_fre_make_run_fremake.py create mode 160000 mkmf diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml index ce5de814..f6ce26c3 100644 --- a/.github/workflows/create_test_conda_env.yml +++ b/.github/workflows/create_test_conda_env.yml @@ -5,66 +5,54 @@ on: [push] jobs: build-linux: runs-on: ubuntu-latest + container: + image: ghcr.io/noaa-gfdl/fre-cli:miniconda24.7.1_gcc14.2.0 steps: - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 with: - python-version: '>=3.9' - - - name: Add conda to system path - run: | - # $CONDA is an env var pointing to root of miniconda dir - echo $CONDA/bin >> $GITHUB_PATH - + submodules: recursive - name: Create fre-cli environment run: | # create environment containing all dependencies # the env cannot be explicitly activated in github CI/CD conda env create -f environment.yml --name fre-cli - # add conda env's executables to github's PATH equiv. + # sets CONDA to wherever it may be on the image + source /root/.bashrc + + # add conda env's executables and mkmf to github's PATH equiv. echo $CONDA/envs/fre-cli/bin >> $GITHUB_PATH - + echo $PWD/mkmf/bin >> $GITHUB_PATH + # use *conda environment's pip* to install fre-cli # called w/ full path to conda's python for explicitness # called as a module (-m pip) for explicitness - $CONDA/envs/fre-cli/bin/python -m pip install --prefix $CONDA/envs/fre-cli . + $CONDA/envs/fre-cli/bin/python -m pip install --prefix $CONDA/envs/fre-cli . - name: Run pytest in fre-cli environment run: | - # try to make sure the right things are in GITHUB_PATH - echo $CONDA/envs/fre-cli/bin >> $GITHUB_PATH - - # are we talking to the right python? - which python - python --version - $CONDA/envs/fre-cli/bin/python --version - + # add spack installed binaries to front of path so that + # conda's netcdf/hdf5 installs don't break compilation tests + export path_save=$PATH + export PATH="/opt/views/view/bin:$PATH" + # run pytest pytest --junit-xml=pytest_results.xml --config-file=fre/pytest.ini --cov-config=fre/coveragerc --cov-report=xml --cov=fre fre/ - - # install genbadge to generate coverage badge based on xml + + # restore original path and install genbadge to generate coverage badge based on xml + export PATH="$path_save" pip install genbadge genbadge coverage -v -i coverage.xml -o docs/cov_badge.svg genbadge tests -v -i pytest_results.xml -o docs/pytest_badge.svg - + - name: Run pylint in fre-cli environment run: | - # try to make sure the right things are in GITHUB_PATH - echo $CONDA/envs/fre-cli/bin >> $GITHUB_PATH - - # are we talking to the right python? - which python - python --version - $CONDA/envs/fre-cli/bin/python --version - - # run pylint, ignored modules avoid warnings arising from code internal to those modules + # run pylint, ignored modules avoid warnings arising from code internal to those modules pylint --max-args 6 -ry --ignored-modules netCDF4,cmor fre/ || echo "pylint returned non-zero exit code. preventing workflow from dying with this echo." - + - name: Install Sphinx and Build Documentation run: | - pip install sphinx renku-sphinx-theme sphinx-rtd-theme + pip install sphinx renku-sphinx-theme sphinx-rtd-theme pip install --upgrade sphinx-rtd-theme sphinx-apidoc --output-dir docs fre/ --separate sphinx-build docs build diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..0ee32c70 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "mkmf"] + path = mkmf + url = https://github.com/NOAA-GFDL/mkmf diff --git a/fre/make/tests/test_fre_make_run_fremake.py b/fre/make/tests/test_fre_make_run_fremake.py new file mode 100644 index 00000000..478bb8b4 --- /dev/null +++ b/fre/make/tests/test_fre_make_run_fremake.py @@ -0,0 +1,13 @@ +''' test "fre make" calls ''' + +from click.testing import CliRunner + +from fre import fre + +runner = CliRunner() + +## assumes you're running pytest from repo root dir +def test_fre_make_run_fremake_null_model(): + ''' run fre make with run-fremake subcommand and build the null model experiment with gnu''' + result = runner.invoke(fre.fre, args=["make", "run-fremake", "-y", "fre/make/tests/null_example/null_model.yaml", "-p", "ci.gnu", "-t", "debug"]) + assert result.exit_code == 0 diff --git a/mkmf b/mkmf new file mode 160000 index 00000000..9830f1ac --- /dev/null +++ b/mkmf @@ -0,0 +1 @@ +Subproject commit 9830f1ac08566ec94e6b28555c921df28b6d0fea From 643ebe0b5cd5526cd57ce9e9072444f777f3c37e Mon Sep 17 00:00:00 2001 From: rem1776 Date: Fri, 1 Nov 2024 13:56:29 -0400 Subject: [PATCH 2/7] add ci platform to null model example --- fre/make/tests/null_example/platforms.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/fre/make/tests/null_example/platforms.yaml b/fre/make/tests/null_example/platforms.yaml index 60d1aad2..fdfa9d4f 100644 --- a/fre/make/tests/null_example/platforms.yaml +++ b/fre/make/tests/null_example/platforms.yaml @@ -24,3 +24,9 @@ platforms: container: True containerBuild: "podman" containerRun: "apptainer" + - name: ci.gnu + compiler: gnu + fc: mpifort + cc: mpicc + mkTemplate: /__w/fre-cli/fre-cli/mkmf/templates/linux-ubuntu-xenial-gnu.mk + modelRoot: ${HOME}/fremake_canopy/test From e9fb43e07c3510404a6e2dcaa2e131273ca8657c Mon Sep 17 00:00:00 2001 From: Ryan Mulhall <35538242+rem1776@users.noreply.github.com> Date: Wed, 6 Nov 2024 10:20:05 -0500 Subject: [PATCH 3/7] finally realized the tests run in both workflows... --- .github/workflows/build_conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index d9ba9162..b08b2814 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -7,7 +7,7 @@ jobs: build: runs-on: ubuntu-latest container: - image: continuumio/miniconda3:latest + image: ghcr.io/noaa-gfdl/fre-cli:miniconda24.7.1_gcc14.2.0 steps: - name: Checkout Files uses: actions/checkout@v4 From 443d4d8ea9f56323e43d7a5c1817695aa182095d Mon Sep 17 00:00:00 2001 From: rem1776 Date: Wed, 6 Nov 2024 10:40:39 -0500 Subject: [PATCH 4/7] add recursive checkouts and mkmf path --- .github/workflows/build_conda.yml | 5 +++++ .github/workflows/publish_conda.yml | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index b08b2814..5ef5e00e 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -11,6 +11,11 @@ jobs: steps: - name: Checkout Files uses: actions/checkout@v4 + with: + submodules: recursive + - name: Add mkmf to PATH + run: | + echo $PWD/mkmf/bin >> $GITHUB_PATH - name: Run Conda to Build run: | conda config --append channels conda-forge diff --git a/.github/workflows/publish_conda.yml b/.github/workflows/publish_conda.yml index e36a72ea..c2c2d622 100644 --- a/.github/workflows/publish_conda.yml +++ b/.github/workflows/publish_conda.yml @@ -7,10 +7,12 @@ jobs: publish: runs-on: ubuntu-latest container: - image: continuumio/miniconda3:latest + image: ghcr.io/noaa-gfdl/fre-cli:miniconda24.7.1_gcc14.2.0 steps: - name: Checkout Files uses: actions/checkout@v4 + with: + submodules: recursive - name: Run Conda to Build and Publish run: | conda config --append channels conda-forge From 081745602dda715769c0e0badf24135a8e44f047 Mon Sep 17 00:00:00 2001 From: rem1776 Date: Wed, 6 Nov 2024 11:03:00 -0500 Subject: [PATCH 5/7] add mkmf path to publish workflow --- .github/workflows/publish_conda.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/publish_conda.yml b/.github/workflows/publish_conda.yml index c2c2d622..b0435703 100644 --- a/.github/workflows/publish_conda.yml +++ b/.github/workflows/publish_conda.yml @@ -13,6 +13,9 @@ jobs: uses: actions/checkout@v4 with: submodules: recursive + - name: Add mkmf to PATH + run: | + echo $PWD/mkmf/bin >> $GITHUB_PATH - name: Run Conda to Build and Publish run: | conda config --append channels conda-forge From 4e5a880680bf9a6465da21af73504462f3b63cc5 Mon Sep 17 00:00:00 2001 From: rem1776 Date: Wed, 6 Nov 2024 15:55:00 -0500 Subject: [PATCH 6/7] make new directory and move run-fremake test so it can be ignored during build --- fre/make/tests/{ => compilation}/test_fre_make_run_fremake.py | 0 meta.yaml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename fre/make/tests/{ => compilation}/test_fre_make_run_fremake.py (100%) diff --git a/fre/make/tests/test_fre_make_run_fremake.py b/fre/make/tests/compilation/test_fre_make_run_fremake.py similarity index 100% rename from fre/make/tests/test_fre_make_run_fremake.py rename to fre/make/tests/compilation/test_fre_make_run_fremake.py diff --git a/meta.yaml b/meta.yaml index a11f0151..878d8042 100644 --- a/meta.yaml +++ b/meta.yaml @@ -63,7 +63,7 @@ test: - fre.catalog commands: - pylint --max-args 6 -ry --ignored-modules netCDF4,cmor fre/ || echo "pylint returned non-zero exit code and will kill the workflow. guarding against this now." - - pip install GitPython && pytest --config-file=fre/pytest.ini --cov-config=fre/coveragerc --cov=fre fre/ + - pip install GitPython && pytest --config-file=fre/pytest.ini --cov-config=fre/coveragerc --cov=fre --ignore=fre/make/tests/compilation fre/ # run pytest but ignore any tests that require compilation - fre --help - fre pp --help - fre pp install --help From 29795a9ac7b072782418a18489f42f7a7cfb907f Mon Sep 17 00:00:00 2001 From: rem1776 Date: Thu, 7 Nov 2024 14:50:35 -0500 Subject: [PATCH 7/7] update run-fremake test file to use a direct function call and check executable exists --- .../compilation/test_fre_make_run_fremake.py | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/fre/make/tests/compilation/test_fre_make_run_fremake.py b/fre/make/tests/compilation/test_fre_make_run_fremake.py index 478bb8b4..be91a547 100644 --- a/fre/make/tests/compilation/test_fre_make_run_fremake.py +++ b/fre/make/tests/compilation/test_fre_make_run_fremake.py @@ -1,13 +1,21 @@ -''' test "fre make" calls ''' +''' test "fre make run-fremake" calls ''' -from click.testing import CliRunner +import os +from fre.make import runFremake +from pathlib import Path -from fre import fre +# command options +YAMLFILE = "fre/make/tests/null_example/null_model.yaml" +PLATFORM = [ "ci.gnu" ] +CONTAINER_PLATFORM = ["hpcme.2023"] +TARGET = ["debug"] +EXPERIMENT = "null_model_full" -runner = CliRunner() +# get HOME dir to check output +HOME_DIR = os.environ["HOME"] -## assumes you're running pytest from repo root dir -def test_fre_make_run_fremake_null_model(): +def test_fre_make_run_fremake_null_model_serial_compile(): ''' run fre make with run-fremake subcommand and build the null model experiment with gnu''' - result = runner.invoke(fre.fre, args=["make", "run-fremake", "-y", "fre/make/tests/null_example/null_model.yaml", "-p", "ci.gnu", "-t", "debug"]) - assert result.exit_code == 0 + runFremake.fremake_run(YAMLFILE, PLATFORM, TARGET, False, 1, False, False) + assert Path(f"{HOME_DIR}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{TARGET[0]}/exec/{EXPERIMENT}.x").exists() +