diff --git a/mache/machines/frontier.cfg b/mache/machines/frontier.cfg index f4bea107..dcd2c8d9 100644 --- a/mache/machines/frontier.cfg +++ b/mache/machines/frontier.cfg @@ -41,7 +41,7 @@ system = slurm parallel_executable = srun # cores per node on the machine -cores_per_node = 56 +cores_per_node = 64 # account for running diagnostics jobs account = cli115 diff --git a/mache/spack/frontier_crayclang_mpich.csh b/mache/spack/frontier_crayclang_mpich.csh new file mode 100644 index 00000000..71b44826 --- /dev/null +++ b/mache/spack/frontier_crayclang_mpich.csh @@ -0,0 +1,20 @@ +module reset >& /dev/null +module switch PrgEnv-cray PrgEnv-cray/8.3.3 >& /dev/null +module switch cce cce/15.0.1 >& /dev/null +module switch craype craype/2.7.20 >& /dev/null + +{% if e3sm_lapack %} +module load cray-libsci/22.12.1.1 +{% endif %} +{% if e3sm_hdf5_netcdf %} +module load cray-hdf5-parallel/1.12.2.1 +module load cray-netcdf-hdf5parallel/4.9.0.1 +module load cray-parallel-netcdf/1.12.3.1 +{% endif %} + +{% if e3sm_hdf5_netcdf %} +setenv NETCDF_C_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX +setenv NETCDF_FORTRAN_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX +setenv PNETCDF_PATH $CRAY_PARALLEL_NETCDF_PREFIX +{% endif %} +setenv HDF5_USE_FILE_LOCKING FALSE diff --git a/mache/spack/frontier_crayclang_mpich.sh b/mache/spack/frontier_crayclang_mpich.sh new file mode 100644 index 00000000..949eef64 --- /dev/null +++ b/mache/spack/frontier_crayclang_mpich.sh @@ -0,0 +1,20 @@ +module reset >& /dev/null +module switch PrgEnv-cray PrgEnv-cray/8.3.3 >& /dev/null +module switch cce cce/15.0.1 >& /dev/null +module switch craype craype/2.7.20 >& /dev/null + +{% if e3sm_lapack %} +module load cray-libsci/22.12.1.1 +{% endif %} +{% if e3sm_hdf5_netcdf %} +module load cray-hdf5-parallel/1.12.2.1 +module load cray-netcdf-hdf5parallel/4.9.0.1 +module load cray-parallel-netcdf/1.12.3.1 +{% endif %} + +{% if e3sm_hdf5_netcdf %} +export NETCDF_C_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX +export NETCDF_FORTRAN_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX +export PNETCDF_PATH=$CRAY_PARALLEL_NETCDF_PREFIX +{% endif %} +export HDF5_USE_FILE_LOCKING=FALSE diff --git a/mache/spack/frontier_crayclang_mpich.yaml b/mache/spack/frontier_crayclang_mpich.yaml new file mode 100644 index 00000000..3fc86d8e --- /dev/null +++ b/mache/spack/frontier_crayclang_mpich.yaml @@ -0,0 +1,175 @@ +spack: + specs: + - cray-mpich +{% if e3sm_lapack %} + - cray-libsci +{% endif %} +{% if e3sm_hdf5_netcdf %} + - hdf5 + - netcdf-c + - netcdf-fortran + - parallel-netcdf +{% endif %} +{{ specs }} + concretizer: + unify: when_possible + packages: + all: + compiler: [cce@15.0.1] + providers: + mpi: [cray-mpich@8.1.23] +{% if e3sm_lapack %} + lapack: [cray-libsci@22.12.1.1] +{% endif %} + autoconf: + externals: + - spec: autoconf@2.69 + prefix: /usr + buildable: false + automake: + externals: + - spec: automake@1.15.1 + prefix: /usr + buildable: false + bzip2: + externals: + - spec: bzip2@1.0.8 + prefix: /usr + buildable: false + cmake: + externals: + - spec: cmake@3.23.2 + prefix: /sw/frontier/spack-envs/base/opt/linux-sles15-x86_64/gcc-7.5.0/cmake-3.23.2-4r4mpiba7cwdw2hlakh5i7tchi64s3qd + buildable: false + curl: + externals: + - spec: curl@7.79.1 + prefix: /usr + buildable: false + gettext: + externals: + - spec: gettext@0.20.2 + prefix: /usr + buildable: false + gmake: + externals: + - spec: gmake@4.3 + prefix: /sw/frontier/spack-envs/base/opt/linux-sles15-x86_64/gcc-7.5.0/gmake-4.3-32qt54ujpdxcat7ceg4vcjlmf2ze25j5 + buildable: false + libxml2: + externals: + - spec: libxml2@2.9.14 + prefix: /usr + buildable: false + m4: + externals: + - spec: m4@1.4.18 + prefix: /usr + buildable: false + ncurses: + externals: + - spec: ncurses@6.1 + prefix: /usr + buildable: false + openssl: + externals: + - spec: openssl@1.1.1l + prefix: /usr + buildable: false + perl: + externals: + - spec: perl@5.26.1 + prefix: /usr + buildable: false + python: + externals: + - spec: python@3.9.13 + prefix: /opt/cray/pe/python/3.9.13.1 + modules: + - cray-python/3.9.13.1 + buildable: false + tar: + externals: + - spec: tar@1.34 + prefix: /usr + buildable: false + xz: + externals: + - spec: xz@5.2.3 + prefix: /usr + buildable: false + cray-mpich: + externals: + - spec: cray-mpich@8.1.23 + prefix: /opt/cray/pe/mpich/8.1.23/ofi/crayclang/10.0 + modules: + - libfabric/1.15.2.0 + - cray-mpich/8.1.23 + buildable: false + libfabric: + externals: + - spec: libfabric@1.15.2.0 + prefix: /opt/cray/libfabric/1.15.2.0 + modules: + - libfabric/1.15.2.0 + buildable: false +{% if e3sm_lapack %} + cray-libsci: + externals: + - spec: cray-libsci@22.12.1.1 + prefix: /opt/cray/pe/libsci/22.12.1.1/crayclang/9.0/x86_64 + modules: + - cray-libsci/22.12.1.1 + buildable: false +{% endif %} +{% if e3sm_hdf5_netcdf or system_hdf5_netcdf %} + hdf5: + externals: + - spec: hdf5@1.12.2.1~cxx+fortran+hl~java+mpi+shared + prefix: /opt/cray/pe/hdf5-parallel/1.12.2.1/crayclang/14.0 + buildable: false + parallel-netcdf: + externals: + - spec: parallel-netcdf@1.12.3.1+cxx+fortran+pic+shared + prefix: /opt/cray/pe/parallel-netcdf/1.12.3.1/crayclang/14.0 + buildable: false + netcdf-c: + externals: + - spec: netcdf-c@4.9.0.1+mpi~parallel-netcdf + prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.1/crayclang/14.0 + buildable: false + netcdf-fortran: + externals: + - spec: netcdf-fortran@4.5.3 + prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.1/crayclang/14.0 + buildable: false +{% endif %} + config: + install_missing_compilers: false + compilers: + - compiler: + spec: cce@15.0.1 + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: sles15 + target: any + modules: + - PrgEnv-cray/8.3.3 + - cce/15.0.1 + - craype/2.7.20 + - libfabric/1.15.2.0 + environment: + set: + RFE_811452_DISABLE: '1' + append_path: + LD_LIBRARY_PATH: /opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + prepend_path: + LD_LIBRARY_PATH: /opt/cray/pe/pmi/6.1.8/lib + PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.22__gd067c3f.shasta/lib64/pkgconfig" + extra_rpaths: + - /opt/cray/pe/gcc-libs + - /opt/cray/gcc-libs diff --git a/mache/spack/frontier_crayclanggpu_mpich.csh b/mache/spack/frontier_crayclanggpu_mpich.csh new file mode 100644 index 00000000..85e79d86 --- /dev/null +++ b/mache/spack/frontier_crayclanggpu_mpich.csh @@ -0,0 +1,22 @@ +module reset >& /dev/null +module switch PrgEnv-cray PrgEnv-cray/8.3.3 >& /dev/null +module switch cce cce/15.0.1 >& /dev/null +module switch craype craype/2.7.20 >& /dev/null +module load craype-accel-amd-gfx90a +module load rocm/5.4.0 + +{% if e3sm_lapack %} +module load cray-libsci/22.12.1.1 +{% endif %} +{% if e3sm_hdf5_netcdf %} +module load cray-hdf5-parallel/1.12.2.1 +module load cray-netcdf-hdf5parallel/4.9.0.1 +module load cray-parallel-netcdf/1.12.3.1 +{% endif %} + +{% if e3sm_hdf5_netcdf %} +setenv NETCDF_C_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX +setenv NETCDF_FORTRAN_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX +setenv PNETCDF_PATH $CRAY_PARALLEL_NETCDF_PREFIX +{% endif %} +setenv HDF5_USE_FILE_LOCKING FALSE diff --git a/mache/spack/frontier_crayclanggpu_mpich.sh b/mache/spack/frontier_crayclanggpu_mpich.sh new file mode 100644 index 00000000..59e664ee --- /dev/null +++ b/mache/spack/frontier_crayclanggpu_mpich.sh @@ -0,0 +1,22 @@ +module reset >& /dev/null +module switch PrgEnv-cray PrgEnv-cray/8.3.3 >& /dev/null +module switch cce cce/15.0.1 >& /dev/null +module switch craype craype/2.7.20 >& /dev/null +module load craype-accel-amd-gfx90a +module load rocm/5.4.0 + +{% if e3sm_lapack %} +module load cray-libsci/22.12.1.1 +{% endif %} +{% if e3sm_hdf5_netcdf %} +module load cray-hdf5-parallel/1.12.2.1 +module load cray-netcdf-hdf5parallel/4.9.0.1 +module load cray-parallel-netcdf/1.12.3.1 +{% endif %} + +{% if e3sm_hdf5_netcdf %} +export NETCDF_C_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX +export NETCDF_FORTRAN_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX +export PNETCDF_PATH=$CRAY_PARALLEL_NETCDF_PREFIX +{% endif %} +export HDF5_USE_FILE_LOCKING=FALSE diff --git a/mache/spack/frontier_crayclanggpu_mpich.yaml b/mache/spack/frontier_crayclanggpu_mpich.yaml new file mode 100644 index 00000000..4115feb4 --- /dev/null +++ b/mache/spack/frontier_crayclanggpu_mpich.yaml @@ -0,0 +1,177 @@ +spack: + specs: + - cray-mpich +{% if e3sm_lapack %} + - cray-libsci +{% endif %} +{% if e3sm_hdf5_netcdf %} + - hdf5 + - netcdf-c + - netcdf-fortran + - parallel-netcdf +{% endif %} +{{ specs }} + concretizer: + unify: when_possible + packages: + all: + compiler: [cce@15.0.1] + providers: + mpi: [cray-mpich@8.1.23] +{% if e3sm_lapack %} + lapack: [cray-libsci@22.12.1.1] +{% endif %} + autoconf: + externals: + - spec: autoconf@2.69 + prefix: /usr + buildable: false + automake: + externals: + - spec: automake@1.15.1 + prefix: /usr + buildable: false + bzip2: + externals: + - spec: bzip2@1.0.8 + prefix: /usr + buildable: false + cmake: + externals: + - spec: cmake@3.23.2 + prefix: /sw/frontier/spack-envs/base/opt/linux-sles15-x86_64/gcc-7.5.0/cmake-3.23.2-4r4mpiba7cwdw2hlakh5i7tchi64s3qd + buildable: false + curl: + externals: + - spec: curl@7.79.1 + prefix: /usr + buildable: false + gettext: + externals: + - spec: gettext@0.20.2 + prefix: /usr + buildable: false + gmake: + externals: + - spec: gmake@4.3 + prefix: /sw/frontier/spack-envs/base/opt/linux-sles15-x86_64/gcc-7.5.0/gmake-4.3-32qt54ujpdxcat7ceg4vcjlmf2ze25j5 + buildable: false + libxml2: + externals: + - spec: libxml2@2.9.14 + prefix: /usr + buildable: false + m4: + externals: + - spec: m4@1.4.18 + prefix: /usr + buildable: false + ncurses: + externals: + - spec: ncurses@6.1 + prefix: /usr + buildable: false + openssl: + externals: + - spec: openssl@1.1.1l + prefix: /usr + buildable: false + perl: + externals: + - spec: perl@5.26.1 + prefix: /usr + buildable: false + python: + externals: + - spec: python@3.9.13 + prefix: /opt/cray/pe/python/3.9.13.1 + modules: + - cray-python/3.9.13.1 + buildable: false + tar: + externals: + - spec: tar@1.34 + prefix: /usr + buildable: false + xz: + externals: + - spec: xz@5.2.3 + prefix: /usr + buildable: false + cray-mpich: + externals: + - spec: cray-mpich@8.1.23 + prefix: /opt/cray/pe/mpich/8.1.23/ofi/crayclang/10.0 + modules: + - libfabric/1.15.2.0 + - cray-mpich/8.1.23 + buildable: false + libfabric: + externals: + - spec: libfabric@1.15.2.0 + prefix: /opt/cray/libfabric/1.15.2.0 + modules: + - libfabric/1.15.2.0 + buildable: false +{% if e3sm_lapack %} + cray-libsci: + externals: + - spec: cray-libsci@22.12.1.1 + prefix: /opt/cray/pe/libsci/22.12.1.1/crayclang/9.0/x86_64 + modules: + - cray-libsci/22.12.1.1 + buildable: false +{% endif %} +{% if e3sm_hdf5_netcdf or system_hdf5_netcdf %} + hdf5: + externals: + - spec: hdf5@1.12.2.1~cxx+fortran+hl~java+mpi+shared + prefix: /opt/cray/pe/hdf5-parallel/1.12.2.1/crayclang/14.0 + buildable: false + parallel-netcdf: + externals: + - spec: parallel-netcdf@1.12.3.1+cxx+fortran+pic+shared + prefix: /opt/cray/pe/parallel-netcdf/1.12.3.1/crayclang/14.0 + buildable: false + netcdf-c: + externals: + - spec: netcdf-c@4.9.0.1+mpi~parallel-netcdf + prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.1/crayclang/14.0 + buildable: false + netcdf-fortran: + externals: + - spec: netcdf-fortran@4.5.3 + prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.1/crayclang/14.0 + buildable: false +{% endif %} + config: + install_missing_compilers: false + compilers: + - compiler: + spec: cce@15.0.1 + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: sles15 + target: any + modules: + - PrgEnv-cray/8.3.3 + - cce/15.0.1 + - craype/2.7.20 + - libfabric/1.15.2.0 + - craype-accel-amd-gfx90a + - rocm/5.4.0 + environment: + set: + RFE_811452_DISABLE: '1' + append_path: + LD_LIBRARY_PATH: /opt/cray/pe/gcc-libs:/opt/cray/libfabric/1.15.2.0/lib64 + prepend_path: + LD_LIBRARY_PATH: /opt/cray/pe/pmi/6.1.8/lib + PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.22__gd067c3f.shasta/lib64/pkgconfig" + extra_rpaths: + - /opt/cray/pe/gcc-libs + - /opt/cray/gcc-libs diff --git a/mache/spack/frontier_gnugpu_mpich.csh b/mache/spack/frontier_gnugpu_mpich.csh new file mode 100644 index 00000000..99554390 --- /dev/null +++ b/mache/spack/frontier_gnugpu_mpich.csh @@ -0,0 +1,21 @@ +module reset >& /dev/null +module switch PrgEnv-cray PrgEnv-gnu/8.3.3 >& /dev/null +module switch gcc gcc/11.2.0 >& /dev/null +module load craype-accel-amd-gfx90a +module load rocm/5.4.0 + +{% if e3sm_lapack %} +module load cray-libsci/22.12.1.1 +{% endif %} +{% if e3sm_hdf5_netcdf %} +module load cray-hdf5-parallel/1.12.2.1 +module load cray-netcdf-hdf5parallel/4.9.0.1 +module load cray-parallel-netcdf/1.12.3.1 +{% endif %} + +{% if e3sm_hdf5_netcdf %} +setenv NETCDF_C_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX +setenv NETCDF_FORTRAN_PATH $CRAY_NETCDF_HDF5PARALLEL_PREFIX +setenv PNETCDF_PATH $CRAY_PARALLEL_NETCDF_PREFIX +{% endif %} +setenv HDF5_USE_FILE_LOCKING FALSE diff --git a/mache/spack/frontier_gnugpu_mpich.sh b/mache/spack/frontier_gnugpu_mpich.sh new file mode 100644 index 00000000..3510567f --- /dev/null +++ b/mache/spack/frontier_gnugpu_mpich.sh @@ -0,0 +1,21 @@ +module reset >& /dev/null +module switch PrgEnv-cray PrgEnv-gnu/8.3.3 >& /dev/null +module switch gcc gcc/11.2.0 >& /dev/null +module load craype-accel-amd-gfx90a +module load rocm/5.4.0 + +{% if e3sm_lapack %} +module load cray-libsci/22.12.1.1 +{% endif %} +{% if e3sm_hdf5_netcdf %} +module load cray-hdf5-parallel/1.12.2.1 +module load cray-netcdf-hdf5parallel/4.9.0.1 +module load cray-parallel-netcdf/1.12.3.1 +{% endif %} + +{% if e3sm_hdf5_netcdf %} +export NETCDF_C_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX +export NETCDF_FORTRAN_PATH=$CRAY_NETCDF_HDF5PARALLEL_PREFIX +export PNETCDF_PATH=$CRAY_PARALLEL_NETCDF_PREFIX +{% endif %} +export HDF5_USE_FILE_LOCKING=FALSE diff --git a/mache/spack/frontier_gnugpu_mpich.yaml b/mache/spack/frontier_gnugpu_mpich.yaml new file mode 100644 index 00000000..2ff58592 --- /dev/null +++ b/mache/spack/frontier_gnugpu_mpich.yaml @@ -0,0 +1,180 @@ +spack: + specs: + - gcc + - cray-mpich +{% if e3sm_lapack %} + - cray-libsci +{% endif %} +{% if e3sm_hdf5_netcdf %} + - hdf5 + - netcdf-c + - netcdf-fortran + - parallel-netcdf +{% endif %} +{{ specs }} + concretizer: + unify: when_possible + packages: + all: + compiler: [gcc@11.2.0] + providers: + mpi: [cray-mpich@8.1.23] +{% if e3sm_lapack %} + lapack: [cray-libsci@22.12.1.1] +{% endif %} + autoconf: + externals: + - spec: autoconf@2.69 + prefix: /usr + buildable: false + automake: + externals: + - spec: automake@1.15.1 + prefix: /usr + buildable: false + bzip2: + externals: + - spec: bzip2@1.0.8 + prefix: /usr + buildable: false + cmake: + externals: + - spec: cmake@3.23.2 + prefix: /sw/frontier/spack-envs/base/opt/linux-sles15-x86_64/gcc-7.5.0/cmake-3.23.2-4r4mpiba7cwdw2hlakh5i7tchi64s3qd + buildable: false + curl: + externals: + - spec: curl@7.79.1 + prefix: /usr + buildable: false + gettext: + externals: + - spec: gettext@0.20.2 + prefix: /usr + buildable: false + gmake: + externals: + - spec: gmake@4.3 + prefix: /sw/frontier/spack-envs/base/opt/linux-sles15-x86_64/gcc-7.5.0/gmake-4.3-32qt54ujpdxcat7ceg4vcjlmf2ze25j5 + buildable: false + libxml2: + externals: + - spec: libxml2@2.9.14 + prefix: /usr + buildable: false + m4: + externals: + - spec: m4@1.4.18 + prefix: /usr + buildable: false + ncurses: + externals: + - spec: ncurses@6.1 + prefix: /usr + buildable: false + openssl: + externals: + - spec: openssl@1.1.1l + prefix: /usr + buildable: false + perl: + externals: + - spec: perl@5.26.1 + prefix: /usr + buildable: false + python: + externals: + - spec: python@3.9.13 + prefix: /opt/cray/pe/python/3.9.13.1 + modules: + - cray-python/3.9.13.1 + buildable: false + tar: + externals: + - spec: tar@1.34 + prefix: /usr + buildable: false + xz: + externals: + - spec: xz@5.2.3 + prefix: /usr + buildable: false + gcc: + externals: + - spec: gcc@11.2.0 + modules: + - PrgEnv-gnu/8.3.3 + - gcc/11.2.0 + - craype/2.7.19 + - libfabric/1.15.2.0 + buildable: false + cray-mpich: + externals: + - spec: cray-mpich@8.1.23 + prefix: /opt/cray/pe/mpich/8.1.23/ofi/gnu/9.1 + modules: + - libfabric/1.15.2.0 + - cray-mpich/8.1.23 + buildable: false + libfabric: + externals: + - spec: libfabric@1.15.2.0 + prefix: /opt/cray/libfabric/1.15.2.0 + modules: + - libfabric/1.15.2.0 + buildable: false +{% if e3sm_lapack %} + cray-libsci: + externals: + - spec: cray-libsci@22.12.1.1 + prefix: /opt/cray/pe/libsci/22.12.1.1/GNU/9.1/x86_64 + modules: + - cray-libsci/22.12.1.1 + buildable: false +{% endif %} +{% if e3sm_hdf5_netcdf or system_hdf5_netcdf %} + hdf5: + externals: + - spec: hdf5@1.12.2.1~cxx+fortran+hl~java+mpi+shared + prefix: /opt/cray/pe/hdf5-parallel/1.12.2.1/GNU/9.1 + buildable: false + parallel-netcdf: + externals: + - spec: parallel-netcdf@1.12.3.1+cxx+fortran+pic+shared + prefix: /opt/cray/pe/parallel-netcdf/1.12.3.1/GNU/9.1/ + buildable: false + netcdf-c: + externals: + - spec: netcdf-c@4.9.0.1+mpi~parallel-netcdf + prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.1/GNU/9.1 + buildable: false + netcdf-fortran: + externals: + - spec: netcdf-fortran@4.5.3 + prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.1/GNU/9.1 + buildable: false +{% endif %} + + config: + install_missing_compilers: false + compilers: + - compiler: + spec: gcc@11.2.0 + paths: + cc: cc + cxx: CC + f77: ftn + fc: ftn + flags: {} + operating_system: sles15 + target: x86_64 + modules: + - PrgEnv-gnu/8.3.3 + - gcc/11.2.0 + - craype/2.7.19 + - libfabric/1.15.2.0 + - craype-accel-amd-gfx90a + - rocm/5.4.0 + environment: + prepend_path: + PKG_CONFIG_PATH: "/opt/cray/xpmem/2.6.2-2.5_2.22__gd067c3f.shasta/lib64/pkgconfig"