Merge pull request #185 from altheaden/add-nvidia-chicoma-cpu

Add nvidia support for `chicoma-cpu`
E3SM-Project · Jul 26, 2024 · 520c085 · 520c085
2 parents 91065b4 + da63595
commit 520c085
Show file tree

Hide file tree

Showing 5 changed files with 260 additions and 2 deletions.
diff --git a/mache/spack/chicoma-cpu_nvidia_mpich.csh b/mache/spack/chicoma-cpu_nvidia_mpich.csh
@@ -0,0 +1,48 @@
+setenv http_proxy http://proxyout.lanl.gov:8080/
+setenv https_proxy http://proxyout.lanl.gov:8080/
+setenv ftp_proxy http://proxyout.lanl.gov:8080
+setenv HTTP_PROXY http://proxyout.lanl.gov:8080
+setenv HTTPS_PROXY http://proxyout.lanl.gov:8080
+setenv FTP_PROXY http://proxyout.lanl.gov:8080
+
+source /usr/share/lmod/8.3.1/init/csh
+
+module rm PrgEnv-gnu
+module rm PrgEnv-nvidia
+module rm PrgEnv-cray
+module rm PrgEnv-aocc
+module rm craype-accel-nvidia80
+module rm craype-accel-host
+
+module load PrgEnv-nvidia/8.4.0
+module load nvidia/22.7
+module load craype-x86-milan
+module load libfabric/1.15.2.0
+module load craype-accel-host
+module load craype
+module load cray-mpich/8.1.26
+{% if e3sm_lapack %}
+module load cray-libsci/23.05.1.4
+{% endif %}
+{% if e3sm_hdf5_netcdf %}
+module rm cray-hdf5-parallel
+module rm cray-netcdf-hdf5parallel
+module rm cray-parallel-netcdf
+module load cray-hdf5-parallel/1.12.2.3
+module load cray-netcdf-hdf5parallel/4.9.0.3
+module load cray-parallel-netcdf/1.12.3.3
+{% endif %}
+
+setenv MPICH_ENV_DISPLAY 1
+setenv MPICH_VERSION_DISPLAY 1
+## purposefully omitting OMP variables that cause trouble in ESMF
+# setenv OMP_STACKSIZE 128M
+# setenv OMP_PROC_BIND spread
+# setenv OMP_PLACES threads
+setenv HDF5_USE_FILE_LOCKING FALSE
+setenv PERL5LIB /usr/projects/climate/SHARED_CLIMATE/software/chicoma-cpu/perl5-only-switch/lib/perl5
+setenv PNETCDF_HINTS "romio_ds_write=disable;romio_ds_read=disable;romio_cb_write=enable;romio_cb_read=enable"
+setenv MPICH_COLL_SYNC MPI_Bcast
+
+setenv LD_LIBRARY_PATH $CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH
+setenv BLA_VENDOR NVHPC
diff --git a/mache/spack/chicoma-cpu_nvidia_mpich.sh b/mache/spack/chicoma-cpu_nvidia_mpich.sh
@@ -0,0 +1,48 @@
+export http_proxy=http://proxyout.lanl.gov:8080/
+export https_proxy=http://proxyout.lanl.gov:8080/
+export ftp_proxy=http://proxyout.lanl.gov:8080
+export HTTP_PROXY=http://proxyout.lanl.gov:8080
+export HTTPS_PROXY=http://proxyout.lanl.gov:8080
+export FTP_PROXY=http://proxyout.lanl.gov:8080
+
+source /usr/share/lmod/8.3.1/init/sh
+
+module rm PrgEnv-gnu
+module rm PrgEnv-nvidia
+module rm PrgEnv-cray
+module rm PrgEnv-aocc
+module rm craype-accel-nvidia80
+module rm craype-accel-host
+
+module load PrgEnv-nvidia/8.4.0
+module load nvidia/22.7
+module load craype-x86-milan
+module load libfabric/1.15.2.0
+module load craype-accel-host
+module load craype
+module load cray-mpich/8.1.26
+{% if e3sm_lapack %}
+module load cray-libsci/23.05.1.4
+{% endif %}
+{% if e3sm_hdf5_netcdf %}
+module rm cray-hdf5-parallel
+module rm cray-netcdf-hdf5parallel
+module rm cray-parallel-netcdf
+module load cray-hdf5-parallel/1.12.2.3
+module load cray-netcdf-hdf5parallel/4.9.0.3
+module load cray-parallel-netcdf/1.12.3.3
+{% endif %}
+
+export MPICH_ENV_DISPLAY=1
+export MPICH_VERSION_DISPLAY=1
+## purposefully omitting OMP variables that cause trouble in ESMF
+# export OMP_STACKSIZE=128M
+# export OMP_PROC_BIND=spread
+# export OMP_PLACES=threads
+export HDF5_USE_FILE_LOCKING=FALSE
+export PERL5LIB=/usr/projects/climate/SHARED_CLIMATE/software/chicoma-cpu/perl5-only-switch/lib/perl5
+export PNETCDF_HINTS="romio_ds_write=disable;romio_ds_read=disable;romio_cb_write=enable;romio_cb_read=enable"
+export MPICH_COLL_SYNC=MPI_Bcast
+
+export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH
+export BLA_VENDOR=NVHPC
diff --git a/mache/spack/chicoma-cpu_nvidia_mpich.yaml b/mache/spack/chicoma-cpu_nvidia_mpich.yaml
@@ -0,0 +1,163 @@
+spack:
+  specs:
+  - cray-mpich
+{% if e3sm_lapack %}
+  - cray-libsci
+{% endif %}
+{% if e3sm_hdf5_netcdf %}
+  - hdf5
+  - netcdf-c
+  - netcdf-fortran
+  - parallel-netcdf
+{% endif %}
+{{ specs }}
+  concretizer:
+    unify: true
+  packages:
+    all:
+      compiler: [[email protected]]
+      providers:
+        mpi: [[email protected]]
+{% if e3sm_lapack %}
+        lapack: [[email protected]]
+{% endif %}
+    bzip2:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    cmake:
+      externals:
+      - spec: [email protected]
+        prefix: /usr/projects/hpcsoft/cos2/common/x86_64/cmake/3.25.1
+      buildable: false
+    curl:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    diffutils:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    gettext:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    gmake:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    libiconv:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    libxml2:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    ncurses:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    openssl:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    perl:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    tar:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    xz:
+      externals:
+      - spec: [email protected]
+        prefix: /usr
+      buildable: false
+    python:
+      externals:
+      - spec: [email protected]
+        prefix: /usr/projects/hpcsoft/common/x86_64/anaconda/2023.03-python-3.10
+        modules:
+        - python/3.10-anaconda-2023.03
+      buildable: false
+    cray-mpich:
+      externals:
+      - spec: [email protected]
+        prefix: /opt/cray/pe/mpich/8.1.26/ofi/nvidia/20.7
+        modules:
+        - libfabric/1.15.2.0
+        - cray-mpich/8.1.26
+      buildable: false
+    libfabric:
+      externals:
+      - spec: [email protected]
+        prefix: /opt/cray/libfabric/1.15.2.0
+        modules:
+        - libfabric/1.15.2.0
+      buildable: false
+{% if e3sm_lapack %}
+    cray-libsci:
+      externals:
+      - spec: [email protected]
+        prefix: /opt/cray/pe/libsci/23.05.1.4/NVIDIA/20.7/x86_64
+      buildable: false
+{% endif %}
+{% if e3sm_hdf5_netcdf or system_hdf5_netcdf %}
+    hdf5:
+      externals:
+      - spec: [email protected]~cxx+fortran+hl~java+mpi+shared
+        prefix: /opt/cray/pe/hdf5-parallel/1.12.2.3/nvidia/20.7
+        modules:
+        - cray-hdf5-parallel/1.12.2.3
+      buildable: false
+    parallel-netcdf:
+      externals:
+      - spec: [email protected]+cxx+fortran+pic+shared
+        prefix: /opt/cray/pe/parallel-netcdf/1.12.3.3/nvidia/20.7
+      buildable: false
+    netcdf-c:
+      externals:
+      - spec: [email protected]+mpi~parallel-netcdf
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/nvidia/20.7
+      buildable: false
+    netcdf-fortran:
+      externals:
+      - spec: [email protected]
+        prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/nvidia/20.7
+      buildable: false
+{% endif %}
+  config:
+    install_missing_compilers: false
+  compilers:
+  - compiler:
+      spec: [email protected]
+      paths:
+        cc: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvc
+        cxx: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvc++
+        f77: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvfortran
+        fc: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvfortran
+      flags: {}
+      operating_system: sles15
+      target: any
+      modules:
+      - PrgEnv-nvidia/8.4.0
+      - nvidia/22.7
+      - craype-x86-milan
+      - craype-accel-host
+      - libfabric/1.15.2.0
+      environment:
+        prepend_path:
+          PKG_CONFIG_PATH: "/opt/cray/xpmem/2.5.2-2.4_3.45__gd0f7936.shasta/lib64/pkgconfig"
diff --git a/mache/spack/pm-cpu_nvidia_mpich.sh b/mache/spack/pm-cpu_nvidia_mpich.sh
@@ -59,7 +59,6 @@ if [ -z "${NERSC_HOST:-}" ]; then
   # happens when building spack environment
   export NERSC_HOST="perlmutter"
 fi
-export MPICH_GPU_SUPPORT_ENABLED=1
 export MPICH_COLL_SYNC=MPI_Bcast
 export GATOR_INITIAL_MB=4000MB
 export BLA_VENDOR=NVHPC
diff --git a/mache/spack/pm-cpu_nvidia_mpich.yaml b/mache/spack/pm-cpu_nvidia_mpich.yaml
@@ -120,7 +120,7 @@ spack:
         prefix: /opt/cray/pe/libsci/23.02.1.1/NVIDIA/20.7/x86_64
       buildable: false
 {% endif %}
-{% if e3sm_hdf5_netcdf or system_hdf5_netcdf%}
+{% if e3sm_hdf5_netcdf or system_hdf5_netcdf %}
     hdf5:
       externals:
       - spec: [email protected]~cxx+fortran+hl~java+mpi+shared