Merge tag 'ctsm5.2.007' into dustemis_dev

Rm manage_externals and update documentation accordingly

Fixes ESCOMP#2537 Remove manage_externals stuff from run_sys_tests
Fixes ESCOMP#2536 Update documentation for git-fleximod
Fixes ESCOMP#2467 Remove references to cheyenne from the testlist

NOTE: Also
Add cam and cice to the .gitmodules file

 Conflicts:
	Externals.cfg

.github/PULL_REQUEST_TEMPLATE.md

@@ -16,7 +16,7 @@ Testing performed, if any:
 (List what testing you did to show your changes worked as expected)
 (This can be manual testing or running of the different test suites)
 (Documentation on system testing is here: https://github.com/ESCOMP/ctsm/wiki/System-Testing-Guide)
-(aux_clm on cheyenne for intel/gnu and izumi for intel/gnu/nag/pgi is the standard for tags on master)
+(aux_clm on derecho for intel/gnu and izumi for intel/gnu/nag/nvhpc is the standard for tags on master)
 
 **NOTE: Be sure to check your coding style against the standard
 (https://github.com/ESCOMP/ctsm/wiki/CTSM-coding-guidelines) and review

.gitignore

@@ -1,14 +1,3 @@
-# directories checked out by manage_externals, and other files created
-# by manage_externals
-manage_externals.log
-ccs_config
-/src/fates/
-/cime/
-/components/
-/libraries/
-/share/
-/doc/doc-builder/
-
 # ignore svn directories
 **/.svn/**
 .svn/

.gitmodules

@@ -0,0 +1,111 @@
+[submodule "fates"]
+path = src/fates
+url = https://github.com/NGEET/fates
+fxtag = sci.1.73.0_api.35.0.0
+fxrequired = AlwaysRequired
+fxDONOTUSEurl = https://github.com/NCAR/fates-release
+
+#
+# CAM and CICE in order to do F compsets along this brunch
+# (Remove before going to master)
+#
+[submodule "cice"]
+	path = components/cice
+	url = https://github.com/ESCOMP/CESM_CICE
+fxtag = cesm_cice6_5_0_9
+fxrequired = ToplevelRequired
+fxDONOTUSEurl = https://github.com/ESCOMP/CESM_CICE
+
+[submodule "cam"]
+path = components/cam
+url = https://github.com/ESCOMP/CAM
+fxtag = cam6_3_162
+fxrequired = ToplevelRequired
+fxDONOTUSEurl = https://github.com/ESCOMP/CAM
+#
+# End
+#
+
+[submodule "cism"]
+path = components/cism
+url = https://github.com/ESCOMP/CISM-wrapper
+fxtag = cismwrap_2_2_001
+fxrequired = ToplevelRequired
+fxDONOTUSEurl = https://github.com/ESCOMP/CISM-wrapper
+
+[submodule "rtm"]
+path = components/rtm
+url = https://github.com/ESCOMP/RTM
+fxtag = rtm1_0_79
+fxrequired = ToplevelRequired
+fxDONOTUSEurl = https://github.com/ESCOMP/RTM
+
+[submodule "mosart"]
+path = components/mosart
+url = https://github.com/ESCOMP/MOSART
+fxtag = mosart1_0_49
+fxrequired = ToplevelRequired
+fxDONOTUSEurl = https://github.com/ESCOMP/MOSART
+
+[submodule "mizuRoute"]
+path = components/mizuRoute
+url = https://github.com/ESCOMP/mizuRoute
+fxtag = cesm-coupling.n02_v2.1.2
+fxrequired = ToplevelRequired
+fxDONOTUSEurl = https://github.com/ESCOMP/mizuRoute
+
+[submodule "ccs_config"]
+path = ccs_config
+url = https://github.com/ESMCI/ccs_config_cesm.git
+fxtag = ccs_config_cesm0.0.106
+fxrequired = ToplevelRequired
+fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git
+
+[submodule "cime"]
+path = cime
+url = https://github.com/ESMCI/cime
+fxtag = cime6.0.246
+fxrequired = ToplevelRequired
+fxDONOTUSEurl = https://github.com/ESMCI/cime
+
+[submodule "cmeps"]
+path = components/cmeps
+url = https://github.com/ESCOMP/CMEPS.git
+fxtag = cmeps0.14.63
+fxrequired = ToplevelRequired
+fxDONOTUSEurl = https://github.com/ESCOMP/CMEPS.git
+
+[submodule "cdeps"]
+path = components/cdeps
+url = https://github.com/ESCOMP/CDEPS.git
+fxtag = cdeps1.0.34
+fxrequired = ToplevelRequired
+fxDONOTUSEurl = https://github.com/ESCOMP/CDEPS.git
+
+[submodule "share"]
+path = share
+url = https://github.com/ESCOMP/CESM_share
+fxtag = share1.0.19
+fxrequired = ToplevelRequired
+fxDONOTUSEurl = https://github.com/ESCOMP/CESM_share
+
+[submodule "mct"]
+path = libraries/mct
+url = https://github.com/MCSclimate/MCT
+fxtag = MCT_2.11.0
+fxrequired = ToplevelRequired
+fxDONOTUSEurl = https://github.com/MCSclimate/MCT
+
+[submodule "parallelio"]
+path = libraries/parallelio
+url = https://github.com/NCAR/ParallelIO
+fxtag = pio2_6_2
+fxrequired = ToplevelRequired
+fxDONOTUSEurl = https://github.com/NCAR/ParallelIO
+
+[submodule "doc-builder"]
+path = doc/doc-builder
+url = https://github.com/ESMCI/doc-builder
+fxtag = v1.0.8
+fxrequired = ToplevelOptional
+fxDONOTUSEurl = https://github.com/ESMCI/doc-builder

.lib/git-fleximod/.github/workflows/pre-commit

@@ -0,0 +1,13 @@
+name: pre-commit
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-python@v3
+    - uses: pre-commit/[email protected]

.lib/git-fleximod/.github/workflows/pytest.yaml

@@ -0,0 +1,77 @@
+# Run this job on pushes to `main`, and for pull requests. If you don't specify
+# `branches: [main], then this actions runs _twice_ on pull requests, which is
+# annoying.
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      # If you wanted to use multiple Python versions, you'd have specify a matrix in the job and
+      # reference the matrixe python version here.
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.9'
+
+      # Cache the installation of Poetry itself, e.g. the next step. This prevents the workflow
+      # from installing Poetry every time, which can be slow. Note the use of the Poetry version
+      # number in the cache key, and the "-0" suffix: this allows you to invalidate the cache
+      # manually if/when you want to upgrade Poetry, or if something goes wrong. This could be
+      # mildly cleaner by using an environment variable, but I don't really care.
+      - name: cache poetry install
+        uses: actions/cache@v4
+        with:
+          path: ~/.local
+          key: poetry-1.7.1
+
+      # Install Poetry. You could do this manually, or there are several actions that do this.
+      # `snok/install-poetry` seems to be minimal yet complete, and really just calls out to
+      # Poetry's default install script, which feels correct. I pin the Poetry version here
+      # because Poetry does occasionally change APIs between versions and I don't want my
+      # actions to break if it does.
+      #
+      # The key configuration value here is `virtualenvs-in-project: true`: this creates the
+      # venv as a `.venv` in your testing directory, which allows the next step to easily
+      # cache it.
+      - uses: snok/install-poetry@v1
+        with:
+          version: 1.7.1
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
+      # Cache your dependencies (i.e. all the stuff in your `pyproject.toml`). Note the cache
+      # key: if you're using multiple Python versions, or multiple OSes, you'd need to include
+      # them in the cache key. I'm not, so it can be simple and just depend on the poetry.lock.
+      - name: cache deps
+        id: cache-deps
+        uses: actions/cache@v4
+        with:
+          path: .venv
+          key: pydeps-${{ hashFiles('**/poetry.lock') }}
+
+      # Install dependencies. `--no-root` means "install all dependencies but not the project
+      # itself", which is what you want to avoid caching _your_ code. The `if` statement
+      # ensures this only runs on a cache miss.
+      - run: poetry install --no-interaction --no-root
+        if: steps.cache-deps.outputs.cache-hit != 'true'
+
+      # Now install _your_ project. This isn't necessary for many types of projects -- particularly
+      # things like Django apps don't need this. But it's a good idea since it fully-exercises the
+      # pyproject.toml and makes that if you add things like console-scripts at some point that
+      # they'll be installed and working.
+      - run: poetry install --no-interaction
+
+      # And finally run tests. I'm using pytest and all my pytest config is in my `pyproject.toml`
+      # so this line is super-simple. But it could be as complex as you need.
+      - run: |
+          git config --global user.name "${GITHUB_ACTOR}"
+          git config --global user.email "${GITHUB_ACTOR_ID}+${GITHUB_ACTOR}@users.noreply.github.com"
+          poetry run pytest
+

.lib/git-fleximod/.pre-commit-config.yaml

@@ -0,0 +1,18 @@
+exclude: ^utils/.*$
+
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.0.1
+    hooks:
+      - id: end-of-file-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/psf/black
+    rev: 22.3.0
+    hooks:
+      - id: black
+  - repo: https://github.com/PyCQA/pylint
+    rev: v2.11.1
+    hooks:
+      - id: pylint
+        args:
+          - --disable=I,C,R,logging-not-lazy,wildcard-import,unused-wildcard-import,fixme,broad-except,bare-except,eval-used,exec-used,global-statement,logging-format-interpolation,no-name-in-module,arguments-renamed,unspecified-encoding,protected-access,import-error,no-member

.lib/git-fleximod/License

@@ -0,0 +1,20 @@
+Copyright 2024 NSF National Center for Atmospheric Sciences (NCAR)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+“Software”), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

.lib/git-fleximod/README.md

@@ -0,0 +1,108 @@
+# git-fleximod
+
+Flexible, Enhanced Submodule Management for Git
+
+## Overview
+
+Git-fleximod is a Python-based tool that extends Git's submodule and sparse checkout capabilities, offering additional features for managing submodules in a more flexible and efficient way.
+
+## Installation
+
+  If you choose to locate git-fleximod in your path you can access it via command: git fleximod
+
+## Usage
+
+    Basic Usage:
+        git fleximod <command> [options]
+    Available Commands:
+        status: Display the status of submodules.
+        update: Update submodules to the tag indicated in .gitmodules variable fxtag.
+        test: Make sure that fxtags and submodule hashes are consistant,
+              make sure that official urls (as defined by fxDONOTUSEurl) are set
+              make sure that fxtags are defined for all submodules
+    Additional Options:
+        See git fleximod --help for more details.
+
+## Supported .gitmodules Variables
+
+    fxtag: Specify a specific tag or branch to checkout for a submodule.
+    fxrequired: Mark a submodule's checkout behavior, with allowed values:
+      - ToplevelRequired: Top-level and required (checked out only when this is the Toplevel module).
+      - ToplevelOptional: Top-level and optional (checked out with --optional flag if this is the Toplevel module).
+      - AlwaysRequired:       Always required (always checked out).
+      - AlwaysOptional:       Always optional (checked out with --optional flag).
+    fxsparse: Enable sparse checkout for a submodule, pointing to a file containing sparse checkout paths.
+    fxDONOTUSEurl: This is the url used in the test subcommand to assure that protected branches do not point to forks
+           **NOTE** the fxDONOTUSEurl variable is only used to identify the official project repository and should not be
+           changed by users.  Use the url variable to change to a fork if desired.
+
+## Sparse Checkouts
+
+    To enable sparse checkout for a submodule, set the fxsparse variable
+    in the .gitmodules file to the path of a file containing the desired
+    sparse checkout paths. Git-fleximod will automatically configure
+    sparse checkout based on this file when applicable commands are run.
+    See [git-sparse-checkout](https://git-scm.com/docs/git-sparse-checkout#_internalsfull_pattern_set)
+    for details on the format of this file.
+
+## Tests
+
+   The git fleximod test action is designed to be used by, for example, github workflows
+   to assure that protected branches are consistant with respect to submodule hashes and fleximod fxtags
+
+## Examples
+
+Here are some common usage examples:
+
+Update all submodules, including optional ones:
+```bash
+  git fleximod update --optional
+```
+
+Updating a specific submodule to the fxtag indicated in .gitmodules:
+
+```bash
+    git fleximod update submodule-name
+```
+Example .gitmodules entry:
+```ini, toml
+    [submodule "cosp2"]
+        path = src/physics/cosp2/src
+        url = https://github.com/CFMIP/COSPv2.0
+        fxsparse = ../.cosp_sparse_checkout
+        fxrequired = AlwaysRequired
+        fxtag = v2.1.4cesm
+```
+Explanation:
+
+This entry indicates that the submodule named cosp2 at tag v2.1.4cesm
+should be checked out into the directory src/physics/cosp2/src
+relative to the .gitmodules directory.  It should be checked out from
+the URL https://github.com/CFMIP/COSPv2.0 and use sparse checkout as
+described in the file ../.cosp_sparse_checkout relative to the path
+directory.  It should be checked out anytime this .gitmodules entry is
+read.
+
+Additional example:
+```ini, toml
+    [submodule "cime"]
+        path = cime
+        url = https://github.com/jedwards4b/cime
+        fxrequired = ToplevelRequired
+        fxtag = cime6.0.198_rme01
+```
+
+Explanation:
+
+This entry indicates that the submodule cime should be checked out
+into a directory named cime at tag cime6.0.198_rme01 from the URL
+https://github.com/jedwards4b/cime.  This should only be done if
+the .gitmodules file is at the top level of the repository clone.
+
+## Contributing
+
+We welcome contributions! Please see the CONTRIBUTING.md file for guidelines.
+
+## License
+
+Git-fleximod is released under the MIT License.

manage_externals/test/doc/Makefile → .lib/git-fleximod/doc/Makefile

@@ -1,10 +1,10 @@
 # Minimal makefile for Sphinx documentation
 #
 
-# You can set these variables from the command line.
-SPHINXOPTS    =
-SPHINXBUILD   = sphinx-build
-SPHINXPROJ    = ManageExternals
+# You can set these variables from the command line, and also
+# from the environment for the first two.
+SPHINXOPTS    ?=
+SPHINXBUILD   ?= sphinx-build
 SOURCEDIR     = .
 BUILDDIR      = _build
 
@@ -17,4 +17,4 @@ help:
 # Catch-all target: route all unknown targets to Sphinx using the new
 # "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
 %: Makefile
-	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)