-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #185 from altheaden/add-nvidia-chicoma-cpu
Add nvidia support for `chicoma-cpu`
- Loading branch information
Showing
5 changed files
with
260 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
setenv http_proxy http://proxyout.lanl.gov:8080/ | ||
setenv https_proxy http://proxyout.lanl.gov:8080/ | ||
setenv ftp_proxy http://proxyout.lanl.gov:8080 | ||
setenv HTTP_PROXY http://proxyout.lanl.gov:8080 | ||
setenv HTTPS_PROXY http://proxyout.lanl.gov:8080 | ||
setenv FTP_PROXY http://proxyout.lanl.gov:8080 | ||
|
||
source /usr/share/lmod/8.3.1/init/csh | ||
|
||
module rm PrgEnv-gnu | ||
module rm PrgEnv-nvidia | ||
module rm PrgEnv-cray | ||
module rm PrgEnv-aocc | ||
module rm craype-accel-nvidia80 | ||
module rm craype-accel-host | ||
|
||
module load PrgEnv-nvidia/8.4.0 | ||
module load nvidia/22.7 | ||
module load craype-x86-milan | ||
module load libfabric/1.15.2.0 | ||
module load craype-accel-host | ||
module load craype | ||
module load cray-mpich/8.1.26 | ||
{% if e3sm_lapack %} | ||
module load cray-libsci/23.05.1.4 | ||
{% endif %} | ||
{% if e3sm_hdf5_netcdf %} | ||
module rm cray-hdf5-parallel | ||
module rm cray-netcdf-hdf5parallel | ||
module rm cray-parallel-netcdf | ||
module load cray-hdf5-parallel/1.12.2.3 | ||
module load cray-netcdf-hdf5parallel/4.9.0.3 | ||
module load cray-parallel-netcdf/1.12.3.3 | ||
{% endif %} | ||
|
||
setenv MPICH_ENV_DISPLAY 1 | ||
setenv MPICH_VERSION_DISPLAY 1 | ||
## purposefully omitting OMP variables that cause trouble in ESMF | ||
# setenv OMP_STACKSIZE 128M | ||
# setenv OMP_PROC_BIND spread | ||
# setenv OMP_PLACES threads | ||
setenv HDF5_USE_FILE_LOCKING FALSE | ||
setenv PERL5LIB /usr/projects/climate/SHARED_CLIMATE/software/chicoma-cpu/perl5-only-switch/lib/perl5 | ||
setenv PNETCDF_HINTS "romio_ds_write=disable;romio_ds_read=disable;romio_cb_write=enable;romio_cb_read=enable" | ||
setenv MPICH_COLL_SYNC MPI_Bcast | ||
|
||
setenv LD_LIBRARY_PATH $CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH | ||
setenv BLA_VENDOR NVHPC |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,48 @@ | ||
export http_proxy=http://proxyout.lanl.gov:8080/ | ||
export https_proxy=http://proxyout.lanl.gov:8080/ | ||
export ftp_proxy=http://proxyout.lanl.gov:8080 | ||
export HTTP_PROXY=http://proxyout.lanl.gov:8080 | ||
export HTTPS_PROXY=http://proxyout.lanl.gov:8080 | ||
export FTP_PROXY=http://proxyout.lanl.gov:8080 | ||
|
||
source /usr/share/lmod/8.3.1/init/sh | ||
|
||
module rm PrgEnv-gnu | ||
module rm PrgEnv-nvidia | ||
module rm PrgEnv-cray | ||
module rm PrgEnv-aocc | ||
module rm craype-accel-nvidia80 | ||
module rm craype-accel-host | ||
|
||
module load PrgEnv-nvidia/8.4.0 | ||
module load nvidia/22.7 | ||
module load craype-x86-milan | ||
module load libfabric/1.15.2.0 | ||
module load craype-accel-host | ||
module load craype | ||
module load cray-mpich/8.1.26 | ||
{% if e3sm_lapack %} | ||
module load cray-libsci/23.05.1.4 | ||
{% endif %} | ||
{% if e3sm_hdf5_netcdf %} | ||
module rm cray-hdf5-parallel | ||
module rm cray-netcdf-hdf5parallel | ||
module rm cray-parallel-netcdf | ||
module load cray-hdf5-parallel/1.12.2.3 | ||
module load cray-netcdf-hdf5parallel/4.9.0.3 | ||
module load cray-parallel-netcdf/1.12.3.3 | ||
{% endif %} | ||
|
||
export MPICH_ENV_DISPLAY=1 | ||
export MPICH_VERSION_DISPLAY=1 | ||
## purposefully omitting OMP variables that cause trouble in ESMF | ||
# export OMP_STACKSIZE=128M | ||
# export OMP_PROC_BIND=spread | ||
# export OMP_PLACES=threads | ||
export HDF5_USE_FILE_LOCKING=FALSE | ||
export PERL5LIB=/usr/projects/climate/SHARED_CLIMATE/software/chicoma-cpu/perl5-only-switch/lib/perl5 | ||
export PNETCDF_HINTS="romio_ds_write=disable;romio_ds_read=disable;romio_cb_write=enable;romio_cb_read=enable" | ||
export MPICH_COLL_SYNC=MPI_Bcast | ||
|
||
export LD_LIBRARY_PATH=$CRAY_LD_LIBRARY_PATH:$LD_LIBRARY_PATH | ||
export BLA_VENDOR=NVHPC |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,163 @@ | ||
spack: | ||
specs: | ||
- cray-mpich | ||
{% if e3sm_lapack %} | ||
- cray-libsci | ||
{% endif %} | ||
{% if e3sm_hdf5_netcdf %} | ||
- hdf5 | ||
- netcdf-c | ||
- netcdf-fortran | ||
- parallel-netcdf | ||
{% endif %} | ||
{{ specs }} | ||
concretizer: | ||
unify: true | ||
packages: | ||
all: | ||
compiler: [[email protected]] | ||
providers: | ||
mpi: [[email protected]] | ||
{% if e3sm_lapack %} | ||
lapack: [[email protected]] | ||
{% endif %} | ||
bzip2: | ||
externals: | ||
- spec: [email protected] | ||
prefix: /usr | ||
buildable: false | ||
cmake: | ||
externals: | ||
- spec: [email protected] | ||
prefix: /usr/projects/hpcsoft/cos2/common/x86_64/cmake/3.25.1 | ||
buildable: false | ||
curl: | ||
externals: | ||
- spec: [email protected] | ||
prefix: /usr | ||
buildable: false | ||
diffutils: | ||
externals: | ||
- spec: [email protected] | ||
prefix: /usr | ||
buildable: false | ||
gettext: | ||
externals: | ||
- spec: [email protected] | ||
prefix: /usr | ||
buildable: false | ||
gmake: | ||
externals: | ||
- spec: [email protected] | ||
prefix: /usr | ||
buildable: false | ||
libiconv: | ||
externals: | ||
- spec: [email protected] | ||
prefix: /usr | ||
buildable: false | ||
libxml2: | ||
externals: | ||
- spec: [email protected] | ||
prefix: /usr | ||
buildable: false | ||
ncurses: | ||
externals: | ||
- spec: [email protected] | ||
prefix: /usr | ||
buildable: false | ||
openssl: | ||
externals: | ||
- spec: [email protected] | ||
prefix: /usr | ||
buildable: false | ||
perl: | ||
externals: | ||
- spec: [email protected] | ||
prefix: /usr | ||
buildable: false | ||
tar: | ||
externals: | ||
- spec: [email protected] | ||
prefix: /usr | ||
buildable: false | ||
xz: | ||
externals: | ||
- spec: [email protected] | ||
prefix: /usr | ||
buildable: false | ||
python: | ||
externals: | ||
- spec: [email protected] | ||
prefix: /usr/projects/hpcsoft/common/x86_64/anaconda/2023.03-python-3.10 | ||
modules: | ||
- python/3.10-anaconda-2023.03 | ||
buildable: false | ||
cray-mpich: | ||
externals: | ||
- spec: [email protected] | ||
prefix: /opt/cray/pe/mpich/8.1.26/ofi/nvidia/20.7 | ||
modules: | ||
- libfabric/1.15.2.0 | ||
- cray-mpich/8.1.26 | ||
buildable: false | ||
libfabric: | ||
externals: | ||
- spec: [email protected] | ||
prefix: /opt/cray/libfabric/1.15.2.0 | ||
modules: | ||
- libfabric/1.15.2.0 | ||
buildable: false | ||
{% if e3sm_lapack %} | ||
cray-libsci: | ||
externals: | ||
- spec: [email protected] | ||
prefix: /opt/cray/pe/libsci/23.05.1.4/NVIDIA/20.7/x86_64 | ||
buildable: false | ||
{% endif %} | ||
{% if e3sm_hdf5_netcdf or system_hdf5_netcdf %} | ||
hdf5: | ||
externals: | ||
- spec: [email protected]~cxx+fortran+hl~java+mpi+shared | ||
prefix: /opt/cray/pe/hdf5-parallel/1.12.2.3/nvidia/20.7 | ||
modules: | ||
- cray-hdf5-parallel/1.12.2.3 | ||
buildable: false | ||
parallel-netcdf: | ||
externals: | ||
- spec: [email protected]+cxx+fortran+pic+shared | ||
prefix: /opt/cray/pe/parallel-netcdf/1.12.3.3/nvidia/20.7 | ||
buildable: false | ||
netcdf-c: | ||
externals: | ||
- spec: [email protected]+mpi~parallel-netcdf | ||
prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/nvidia/20.7 | ||
buildable: false | ||
netcdf-fortran: | ||
externals: | ||
- spec: [email protected] | ||
prefix: /opt/cray/pe/netcdf-hdf5parallel/4.9.0.3/nvidia/20.7 | ||
buildable: false | ||
{% endif %} | ||
config: | ||
install_missing_compilers: false | ||
compilers: | ||
- compiler: | ||
spec: [email protected] | ||
paths: | ||
cc: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvc | ||
cxx: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvc++ | ||
f77: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvfortran | ||
fc: /opt/nvidia/hpc_sdk/Linux_x86_64/22.7/compilers/bin/nvfortran | ||
flags: {} | ||
operating_system: sles15 | ||
target: any | ||
modules: | ||
- PrgEnv-nvidia/8.4.0 | ||
- nvidia/22.7 | ||
- craype-x86-milan | ||
- craype-accel-host | ||
- libfabric/1.15.2.0 | ||
environment: | ||
prepend_path: | ||
PKG_CONFIG_PATH: "/opt/cray/xpmem/2.5.2-2.4_3.45__gd0f7936.shasta/lib64/pkgconfig" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -120,7 +120,7 @@ spack: | |
prefix: /opt/cray/pe/libsci/23.02.1.1/NVIDIA/20.7/x86_64 | ||
buildable: false | ||
{% endif %} | ||
{% if e3sm_hdf5_netcdf or system_hdf5_netcdf%} | ||
{% if e3sm_hdf5_netcdf or system_hdf5_netcdf %} | ||
hdf5: | ||
externals: | ||
- spec: [email protected]~cxx+fortran+hl~java+mpi+shared | ||
|