Merge pull request #245 from rem1776/update-ci-for-compilation

update CI image and add mkmf for fre make testing

.github/workflows/build_conda.yml

@@ -7,12 +7,15 @@ jobs:
   build:
     runs-on: ubuntu-latest
     container:
-      image: continuumio/miniconda3:latest
+      image: ghcr.io/noaa-gfdl/fre-cli:miniconda24.7.1_gcc14.2.0
     steps:
       - name: Checkout Files
         uses: actions/checkout@v4
         with:
           submodules: 'recursive'
+      - name: Add mkmf to PATH
+        run: |
+          echo $PWD/mkmf/bin >> $GITHUB_PATH
       - name: Run Conda to Build
         run: |
           conda config --append channels conda-forge

.github/workflows/create_test_conda_env.yml

@@ -5,27 +5,24 @@ on: [push]
 jobs:
   build-linux:
     runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/noaa-gfdl/fre-cli:miniconda24.7.1_gcc14.2.0
     steps:
       - uses: actions/checkout@v4
         with:
           submodules: 'recursive'
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: '>=3.9'
-      - name: Add conda to system path
-        run: |
-          # $CONDA is an env var pointing to root of miniconda dir
-          echo $CONDA/bin >> $GITHUB_PATH
-
       - name: Create fre-cli environment
         run: |
           # create environment containing all dependencies
           # the env cannot be explicitly activated in github CI/CD
           conda env create -f environment.yml --name fre-cli
 
-          # add conda env's executables to github's PATH equiv.
+          # sets CONDA to wherever it may be on the image
+          source /root/.bashrc
+
+          # add conda env's executables and mkmf to github's PATH equiv.
           echo $CONDA/envs/fre-cli/bin >> $GITHUB_PATH
+          echo $PWD/mkmf/bin >> $GITHUB_PATH
 
           # use *conda environment's pip* to install fre-cli
           # called w/ full path to conda's python for explicitness
@@ -34,38 +31,28 @@ jobs:
 
       - name: Run pytest in fre-cli environment
         run: |
-          # try to make sure the right things are in GITHUB_PATH
-          echo $CONDA/envs/fre-cli/bin >> $GITHUB_PATH
-          
-          # are we talking to the right python?
-          which python
-          python --version
-          $CONDA/envs/fre-cli/bin/python --version
-          
+          # add spack installed binaries to front of path so that
+          # conda's netcdf/hdf5 installs don't break compilation tests
+          export path_save=$PATH
+          export PATH="/opt/views/view/bin:$PATH"
+
           # run pytest
           pytest --junit-xml=pytest_results.xml --config-file=fre/pytest.ini --cov-config=fre/coveragerc --cov-report=xml --cov=fre fre/
-          
-          # install genbadge to generate coverage badge based on xml
+
+          # restore original path and install genbadge to generate coverage badge based on xml
+          export PATH="$path_save"
           pip install genbadge
           genbadge coverage -v -i coverage.xml -o docs/cov_badge.svg
           genbadge tests -v -i pytest_results.xml -o docs/pytest_badge.svg
-        
+
       - name: Run pylint in fre-cli environment
         run: |
-          # try to make sure the right things are in GITHUB_PATH
-          echo $CONDA/envs/fre-cli/bin >> $GITHUB_PATH
-
-          # are we talking to the right python?
-          which python
-          python --version
-          $CONDA/envs/fre-cli/bin/python --version
-          
-          # run pylint, ignored modules avoid warnings arising from code internal to those modules 
+          # run pylint, ignored modules avoid warnings arising from code internal to those modules
           pylint --max-args 6 -ry --ignored-modules netCDF4,cmor fre/ || echo "pylint returned non-zero exit code. preventing workflow from dying with this echo."
 
       - name: Install Sphinx and Build Documentation
         run: |
-          pip install sphinx renku-sphinx-theme sphinx-rtd-theme          
+          pip install sphinx renku-sphinx-theme sphinx-rtd-theme
           pip install --upgrade sphinx-rtd-theme
           sphinx-apidoc --output-dir docs fre/ --separate
           sphinx-build docs build

.github/workflows/publish_conda.yml

@@ -7,12 +7,15 @@ jobs:
   publish:
     runs-on: ubuntu-latest
     container:
-      image: continuumio/miniconda3:latest
+      image: ghcr.io/noaa-gfdl/fre-cli:miniconda24.7.1_gcc14.2.0
     steps:
       - name: Checkout Files
         uses: actions/checkout@v4
         with:
           submodules: 'recursive'
+      - name: Add mkmf to PATH
+        run: |
+          echo $PWD/mkmf/bin >> $GITHUB_PATH
       - name: Run Conda to Build and Publish
         run: |
           conda config --append channels conda-forge

.gitmodules

@@ -1,3 +1,6 @@
+[submodule "mkmf"]
+	path = mkmf
+	url = https://github.com/NOAA-GFDL/mkmf
 [submodule "fre/gfdl_msd_schemas"]
 	path = fre/gfdl_msd_schemas
 	url = https://github.com/NOAA-GFDL/gfdl_msd_schemas

fre/make/tests/compilation/test_fre_make_run_fremake.py

@@ -0,0 +1,21 @@
+''' test "fre make run-fremake" calls '''
+
+import os
+from fre.make import runFremake
+from pathlib import Path
+
+# command options
+YAMLFILE = "fre/make/tests/null_example/null_model.yaml"
+PLATFORM = [ "ci.gnu" ]
+CONTAINER_PLATFORM = ["hpcme.2023"]
+TARGET = ["debug"]
+EXPERIMENT = "null_model_full"
+
+# get HOME dir to check output
+HOME_DIR = os.environ["HOME"]
+
+def test_fre_make_run_fremake_null_model_serial_compile():
+    ''' run fre make with run-fremake subcommand and build the null model experiment with gnu'''
+    runFremake.fremake_run(YAMLFILE, PLATFORM, TARGET, False, 1, False, False)
+    assert Path(f"{HOME_DIR}/fremake_canopy/test/{EXPERIMENT}/{PLATFORM[0]}-{TARGET[0]}/exec/{EXPERIMENT}.x").exists()
+

fre/make/tests/null_example/platforms.yaml

@@ -24,3 +24,9 @@ platforms:
      container: True
      containerBuild: "podman"
      containerRun: "apptainer" 
+   - name: ci.gnu
+     compiler: gnu
+     fc: mpifort
+     cc: mpicc
+     mkTemplate: /__w/fre-cli/fre-cli/mkmf/templates/linux-ubuntu-xenial-gnu.mk
+     modelRoot: ${HOME}/fremake_canopy/test

meta.yaml

@@ -63,7 +63,7 @@ test:
     - fre.catalog
   commands:
     - pylint --max-args 6 -ry --ignored-modules netCDF4,cmor fre/ || echo "pylint returned non-zero exit code and will kill the workflow. guarding against this now."
-    - pip install GitPython && pytest --config-file=fre/pytest.ini --cov-config=fre/coveragerc --cov=fre fre/
+    - pip install GitPython && pytest --config-file=fre/pytest.ini --cov-config=fre/coveragerc --cov=fre --ignore=fre/make/tests/compilation fre/ # run pytest but ignore any tests that require compilation
     - fre --help
     - fre pp --help
     - fre pp install --help