Merge pull request #33 from NOAA-GFDL/conda_env_attempt

Conda env attempt

.github/workflows/create_test_conda_env.yml

@@ -0,0 +1,74 @@
+name: create_test_conda_env
+
+on: [push]
+
+jobs:
+  build-linux:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '>=3.9'
+
+      - name: Add conda to system path
+        run: |
+          # $CONDA is an env var pointing to root of miniconda dir
+          echo $CONDA/bin >> $GITHUB_PATH
+
+      - name: Create fre-workflows environment
+        run: |
+          # create environment containing all dependencies
+          # the env cannot be explicitly activated in github CI/CD
+          conda env create -f environment.yml --name fre-workflows
+
+          # add conda env's executables to github's PATH equiv.
+          echo $CONDA/envs/fre-workflows/bin >> $GITHUB_PATH
+
+      - name: in fre-workflows environment, cylc lint
+        continue-on-error: true
+        run: |
+          # lint .cylc workflow files
+          cylc lint -v
+
+      - name: in fre-workflows environment, data_lineage unittest and pylint
+        continue-on-error: true
+        run: |
+          # unittests for data_lineage
+          cd data_lineage && \
+             python -m unittest discover -s test -v ; \
+             pylint -v ; \
+             cd - ;
+
+      - name: in fre-workflows environment, regrid-xy pytest and pylint
+        continue-on-error: true
+        run: |
+          # pytest unittests for regrid-xy
+          cd app/regrid-xy && \
+             pytest -v -v -rx --cov=regrid_xy ./t ; \
+             pylint -v ; \
+             cd - ;
+
+      - name: in fre-workflows environment, make-timeseries pytest and pylint
+        continue-on-error: true
+        run: |
+          # pytest unittests for make-timeseries
+          cd app/make-timeseries && \
+             pytest -v -v -rx --cov=bin/make-timeseries ./test ; \
+             pylint -v ; \
+             cd - ;
+
+      - name: in fre-workflows environment, remap-pp-components pytest and pylint
+        continue-on-error: true
+        run: |
+          # pytest unittests for remap-pp-components
+          cd app/remap-pp-components && \
+             pytest -v -v -rx --cov=bin/remap-pp-components ./t ; \
+             cd - ;
+
+      - name: in fre-workflows environment, root directory pytest
+        continue-on-error: true
+        run: |
+          # pytest unittests for other things in this repository
+          pytest -v -v -rx ./tests

.gitignore

@@ -1,2 +1,3 @@
 __pycache__
 *~
+tests/test_files_papiex_tooler/*tags

environment.yml

@@ -0,0 +1,20 @@
+name: fre-workflows
+channels:
+  - defaults
+  - conda-forge
+  - noaa-gfdl
+dependencies:
+  - python
+  - pip
+  - noaa-gfdl::fre-cli
+#  - click
+#  - pyyaml
+#  - pylint
+#  - jsonschema
+#  - noaa-gfdl::fre-nctools
+#  - conda-forge::cylc-flow>=8.2.0
+#  - conda-forge::cylc-rose
+#  - conda-forge::metomi-rose
+#  - conda-forge::cylc-uiserver
+#  - conda-forge::pytest
+#  - conda-forge::pytest-cov

tests/test_PPANHandler.py

@@ -1,30 +1,30 @@
 ''' tests for PP/AN specific job_runner_handler class '''
 
-def test_import():
+def test_import(capfd):
     ''' check that ppan_handler can be imported.'''
     #print(f'__name__=={__name__}')
     from lib.python.ppan_handler import PPANHandler
     test_handler=PPANHandler()
 
-    assert(test_handler.test_import() == 0)
+    assert test_handler.test_import() == 0
 
 
-def test_tool_ops_import_in_handler():
+def test_tool_ops_import_in_handler(capfd):
     ''' check that ppan_handler can import tool_ops_w_papiex'''
     #print(f'__name__=={__name__}')
     from lib.python.ppan_handler import PPANHandler
     test_handler=PPANHandler()
 
-    assert(test_handler.test_tool_ops_import() == 0)
+    assert test_handler.test_tool_ops_import() == 0
 
-def test_submit():
+def test_submit(capfd):
     ''' check ppan_handler submit behavior with dry_run=True '''
     #print(f'__name__=={__name__}')
     from lib.python.ppan_handler import PPANHandler
     test_handler=PPANHandler()
 
     from pathlib import Path
-    job_file_path='./tests/test_files_papiex_tooler/am5_c96L33_amip_job_stage-history'
+    job_file_path='./tests/test_files_papiex_tooler/am5_c96L33_amip_mask-atmos-plevel_atmos_scalar_job'
 
     # check for existing test output, remove in order to recreate
     if Path(job_file_path+'.notags').exists():
@@ -37,10 +37,10 @@ def test_submit():
                                    job_file_path = job_file_path,
                                    submit_opts = submit_opts,
                                    dry_run = True, tool_ops = True)
-    assert(all([ ret_code == 0,
-                 ret_out  == "HELLO\n",
-                 ret_err  == "",
-                 Path(job_file_path+'.notags').exists(),
-                 Path(job_file_path).exists()  ]))
+    assert all ( [ ret_code == 0,
+                   ret_out  == "HELLO\n",
+                   ret_err  == "",
+                   Path(job_file_path+'.notags').exists(),
+                   Path(job_file_path).exists() ] ) 
 
 

tests/test_papiex_tooler.py

@@ -1,29 +1,32 @@
 ''' tests for PP/AN specific ops-tooling of job scripts for PAPIEX'''
+from subprocess import Popen, PIPE, DEVNULL
+from pathlib import Path
+import filecmp as fc
+import difflib as dl
 import pytest
 
 def test_import():
     ''' check that tool_ops_w_papiex functions can be imported '''
     from  lib.python.tool_ops_w_papiex import test_import
-    assert(test_import() == 0)
+    assert test_import() == 0
 
 def test_import_papiex_ops():
     ''' check that op list data can be imported '''
     import lib.python.papiex_ops as po
-    assert ( all( [ po.op_list is not None,
-                       len(po.op_list) > 0  ] ) )
+    assert all( [ po.op_list is not None,
+                     len(po.op_list) > 0  ] ) 
 
 def test_simple_failing_command():
     ''' check that setting/unsetting PAPIEX_TAGS around an op
     will not touch the exit status of that op '''
-    from pathlib import Path
-    control_script_targ=str(Path.cwd())+'/tests/test_files_papiex_tooler/simple_failing_command.bash'
+    control_script_targ = str(Path.cwd()) + \
+        '/tests/test_files_papiex_tooler/simple_failing_command.bash'
     assert Path(control_script_targ).exists() #quick check
 
     # import subprocess and run "control" process for later comparison
-    from subprocess import Popen, PIPE, DEVNULL
-    control_proc=None
+    control_proc = None
     try:
-        control_proc=Popen( args=['/bin/bash' ,control_script_targ],
+        control_proc = Popen( args=['/bin/bash' ,control_script_targ],
                          bufsize=0,
                          executable=None,
                          stdin=DEVNULL, stdout=PIPE, stderr=PIPE,
@@ -33,13 +36,14 @@ def test_simple_failing_command():
         assert False
 
     # grab control output
-    control_out, control_err= None,None
-    control_ret_code=None
+    control_out, control_err = None, None
+    control_ret_code = None
     try:
         control_out, control_err = (
             f.decode() for f in control_proc.communicate(DEVNULL) )
         control_ret_code = control_proc.wait()
-        print(f'control_out, control_err, control_ret_code = \n {control_out}, \n {control_err}, \n {control_ret_code}')
+        print(f'control_out, control_err, control_ret_code = ' + \
+              '\n {control_out}, \n {control_err}, \n {control_ret_code}')
         assert all( [ control_out is not None,
                       control_err is not None,
                       control_ret_code is not None ] )        
@@ -94,20 +98,17 @@ def test_check_simple_failing_command_for_diff():
     prev test, which could in theory succeed if tool_ops_w_papiex copies 
     the script but without adding tooling.'''
 
-    from pathlib import Path
     control_script_targ=str(Path.cwd())+'/tests/test_files_papiex_tooler/simple_failing_command.bash'
     script_targ=control_script_targ+'.tags'
     assert Path(control_script_targ).exists() #quick check
     assert Path(script_targ).exists() #quick check
 
     # check quickly that they are different in some manner.
-    import filecmp as fc
     is_different=not fc.cmp( control_script_targ, script_targ,
                              shallow=False)
     print(f'different? {is_different}\n\n')
 
     # now we will explicitly check for those differencves
-    import difflib as dl
     the_infile = open(control_script_targ)
     infile_contents=the_infile.readlines()
     the_infile.close()
@@ -135,8 +136,8 @@ def test_check_simple_failing_command_for_diff():
 
 
 def test_rose_task_run_for_diff():    
-    from pathlib import Path
-    control_script_targ=str(Path.cwd())+'/tests/test_files_papiex_tooler/am5_c96L33_amip_mask-atmos-plevel_atmos_scalar_job'
+    control_script_targ = str(Path.cwd()) + \
+        '/tests/test_files_papiex_tooler/am5_c96L33_amip_mask-atmos-plevel_atmos_scalar_job'
 
 
     # if we're testing over and over and '.notags' version exists,
@@ -159,13 +160,11 @@ def test_rose_task_run_for_diff():
     assert Path(script_targ).exists() #quick check
 
     # check quickly that they are different in some manner.
-    import filecmp as fc
     is_different=not fc.cmp( control_script_targ, script_targ,
                              shallow=False)
     print(f'different? {is_different}\n\n')
 
     # now we will explicitly check for those differences
-    import difflib as dl
     the_infile = open(control_script_targ)
     infile_contents=the_infile.readlines()
     the_infile.close()
@@ -197,7 +196,6 @@ def test_rose_task_run_for_diff():
 
 
 def test_pp_starter_for_no_diff():    
-    from pathlib import Path
     control_script_targ=str(Path.cwd())+'/tests/test_files_papiex_tooler/test_pp-starter'
 
     # if we're testing over and over and '.notags' version exists,
@@ -220,13 +218,11 @@ def test_pp_starter_for_no_diff():
     assert Path(script_targ).exists() #quick check
 
     # check quickly that they are different in some manner.
-    import filecmp as fc
     is_different=not fc.cmp( control_script_targ, script_targ,
                              shallow=False)
     print(f'different? {is_different}\n\n')
 
     # now we will explicitly check for those differences
-    import difflib as dl
     the_infile = open(control_script_targ)
     infile_contents=the_infile.readlines()
     the_infile.close()