diff --git a/repo/package_diffs_vs_spack_0.17.txt b/repo/package_diffs_vs_spack_0.17.txt index 53e0e5ce..be638738 100644 --- a/repo/package_diffs_vs_spack_0.17.txt +++ b/repo/package_diffs_vs_spack_0.17.txt @@ -4455,3 +4455,8 @@ Differences for the 'wcstools' package Differences for the 'wsclean' package +Differences for the 'singularityce' package +- is primarily the correct handling of the conmon dependency/build variant and also hash for new versions + +Differences for the 'go' package +- just the addition of newer versions and also pulled from spack dev as of 01/07/2023 as basis to add lots of versions. \ No newline at end of file diff --git a/repo/packages/go/misc-cgo-testcshared.patch b/repo/packages/go/misc-cgo-testcshared.patch new file mode 100644 index 00000000..17751df8 --- /dev/null +++ b/repo/packages/go/misc-cgo-testcshared.patch @@ -0,0 +1,11 @@ +--- misc/cgo/testcshared/test.bash.orig 2016-11-19 00:00:11.917000000 +0000 ++++ misc/cgo/testcshared/test.bash 2016-11-19 00:00:22.081000000 +0000 +@@ -107,7 +107,7 @@ + + # test0: exported symbols in shared lib are accessible. + # TODO(iant): using _shared here shouldn't really be necessary. +-$(go env CC) ${GOGCCFLAGS} -I ${installdir} -o testp main0.c libgo.$libext ++$(go env CC) ${GOGCCFLAGS} -I ${installdir} -o testp main0.c ./libgo.$libext + binpush testp + + output=$(run LD_LIBRARY_PATH=. ./testp) diff --git a/repo/packages/go/package.py b/repo/packages/go/package.py new file mode 100644 index 00000000..79867f91 --- /dev/null +++ b/repo/packages/go/package.py @@ -0,0 +1,205 @@ +# just added a version +########################################## +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os +import platform +import re + +import llnl.util.tty as tty + +from spack.package import * + +# - vanilla CentOS 7, and possibly other systems, fail a test: +# TestCloneNEWUSERAndRemapRootDisableSetgroups +# +# The Fix, discussed here: https://github.com/golang/go/issues/16283 +# is to enable "user_namespace". +# +# On a Digital Ocean image, this can be achieved by updating +# `/etc/default/grub` so that the `GRUB_CMDLINE_LINUX` variable +# includes `user_namespace.enable=1`, re-cooking the grub +# configuration with `sudo grub2-mkconfig -o /boot/grub2/grub.cfg`, +# and then rebooting. +# +# - on CentOS 7 systems (and possibly others) you need to have the +# glibc package installed or various static cgo tests fail. +# +# - When building on a *large* machine (144 cores, 1.5TB RAM) I need +# to run `ulimit -u 8192` to bump up the max number of user processes. +# Failure to do so results in an explosion in one of the tests and an +# epic stack trace.... + + +class Go(Package): + """The golang compiler and build environment""" + + homepage = "https://golang.org" + url = "https://dl.google.com/go/go1.16.6.src.tar.gz" + git = "https://go.googlesource.com/go.git" + + extendable = True + executables = ["^go$"] + + maintainers = ["alecbcs"] + + version("1.20.4", sha256="9f34ace128764b7a3a4b238b805856cc1b2184304df9e5690825b0710f4202d6") + version("1.20.3", sha256="e447b498cde50215c4f7619e5124b0fc4e25fb5d16ea47271c47f278e7aa763a") + + version("1.19.10", sha256="13755bcce529747d5f2930dee034730c86d02bd3e521ab3e2bbede548d3b953f") + version("1.19.9", sha256="131190a4697a70c5b1d232df5d3f55a3f9ec0e78e40516196ffb3f09ae6a5744") + version("1.19.8", sha256="1d7a67929dccafeaf8a29e55985bc2b789e0499cb1a17100039f084e3238da2f") + + # Deprecated Versions + # https://nvd.nist.gov/vuln/detail/CVE-2023-24538 + version( + "1.20.2", + sha256="4d0e2850d197b4ddad3bdb0196300179d095bb3aefd4dfbc3b36702c3728f8ab", + deprecated=True, + ) + version( + "1.19.7", + sha256="775bdf285ceaba940da8a2fe20122500efd7a0b65dbcee85247854a8d7402633", + deprecated=True, + ) + + # https://nvd.nist.gov/vuln/detail/CVE-2023-24532 + version( + "1.20.1", + sha256="b5c1a3af52c385a6d1c76aed5361cf26459023980d0320de7658bae3915831a2", + deprecated=True, + ) + # https://nvd.nist.gov/vuln/detail/CVE-2022-41723 + version( + "1.20", + sha256="3a29ff0421beaf6329292b8a46311c9fbf06c800077ceddef5fb7f8d5b1ace33", + deprecated=True, + ) + # https://nvd.nist.gov/vuln/detail/CVE-2022-41725 + version( + "1.19.6", + sha256="d7f0013f82e6d7f862cc6cb5c8cdb48eef5f2e239b35baa97e2f1a7466043767", + deprecated=True, + ) + # https://nvd.nist.gov/vuln/detail/CVE-2022-41725 + version( + "1.19.5", + sha256="8e486e8e85a281fc5ce3f0bedc5b9d2dbf6276d7db0b25d3ec034f313da0375f", + deprecated=True, + ) + version( + "1.19.4", + sha256="eda74db4ac494800a3e66ee784e495bfbb9b8e535df924a8b01b1a8028b7f368", + deprecated=True, + ) + # https://nvd.nist.gov/vuln/detail/CVE-2022-41724 + version( + "1.18.10", + sha256="9cedcca58845df0c9474ae00274c44a95c9dfaefb132fc59921c28c7c106f8e6", + deprecated=True, + ) + version( + "1.18.9", + sha256="fbe7f09b96aca3db6faeaf180da8bb632868ec049731e355ff61695197c0e3ea", + deprecated=True, + ) + + provides("golang") + + depends_on("git", type=("build", "link", "run")) + + # aarch64 machines (including Macs with Apple silicon) can't use + # go-bootstrap because it pre-dates aarch64 support in Go. These machines + # have to rely on Go support in gcc (which may require compiling a version + # of gcc with Go support just to satisfy this requirement) or external go: + + # #27769: On M1/MacOS, platform.machine() may return arm64: + if platform.machine() in ["arm64", "aarch64"]: + # Use an external go compiler from packages.yaml/`spack external find go-bootstrap`, + # but fallback to build go-bootstrap@1.4 or to gcc with languages=go (for aarch64): + depends_on("go-external-or-gccgo-bootstrap", type="build") + else: + depends_on("go-bootstrap", type="build") + + # https://github.com/golang/go/issues/17545 + patch("time_test.patch", when="@1.6.4:1.7.4") + + # https://github.com/golang/go/issues/17986 + # The fix for this issue has been merged into the 1.8 tree. + patch("misc-cgo-testcshared.patch", level=0, when="@1.6.4:1.7.5") + + # Unrecognized option '-fno-lto' + conflicts("%gcc@:4", when="@1.17:") + + @classmethod + def determine_version(cls, exe): + output = Executable(exe)("version", output=str, error=str) + match = re.search(r"go version go(\S+)", output) + return match.group(1) if match else None + + # NOTE: Older versions of Go attempt to download external files that have + # since been moved while running the test suite. This patch modifies the + # test files so that these tests don't cause false failures. + # See: https://github.com/golang/go/issues/15694 + @when("@:1.4.3") + def patch(self): + test_suite_file = FileFilter(join_path("src", "run.bash")) + test_suite_file.filter( + r"^(.*)(\$GOROOT/src/cmd/api/run.go)(.*)$", + r"# \1\2\3", + ) + + def install(self, spec, prefix): + bash = which("bash") + + wd = "." + + # 1.11.5 directory structure is slightly different + if self.version == Version("1.11.5"): + wd = "go" + + with working_dir(join_path(wd, "src")): + bash("{0}.bash".format("all" if self.run_tests else "make")) + + install_tree(wd, prefix) + + def setup_build_environment(self, env): + env.set("GOROOT_FINAL", self.spec.prefix) + # We need to set CC/CXX_FOR_TARGET, otherwise cgo will use the + # internal Spack wrappers and fail. + env.set("CC_FOR_TARGET", self.compiler.cc) + env.set("CXX_FOR_TARGET", self.compiler.cxx) + + def setup_dependent_package(self, module, dependent_spec): + """Called before go modules' install() methods. + + In most cases, extensions will only need to set GOPATH and use go:: + + env['GOPATH'] = self.source_path + ':' + env['GOPATH'] + go('get', '', env=env) + install_tree('bin', prefix.bin) + """ + # Add a go command/compiler for extensions + module.go = self.spec["go"].command + + def generate_path_components(self, dependent_spec): + if os.environ.get("GOROOT", False): + tty.warn("GOROOT is set, this is not recommended") + + # Set to include paths of dependencies + path_components = [dependent_spec.prefix] + for d in dependent_spec.traverse(): + if d.package.extends(self.spec): + path_components.append(d.prefix) + return ":".join(path_components) + + def setup_dependent_build_environment(self, env, dependent_spec): + # This *MUST* be first, this is where new code is installed + env.prepend_path("GOPATH", self.generate_path_components(dependent_spec)) + + def setup_dependent_run_environment(self, env, dependent_spec): + # Allow packages to find this when using module files + env.prepend_path("GOPATH", self.generate_path_components(dependent_spec)) diff --git a/repo/packages/go/package.py.old b/repo/packages/go/package.py.old new file mode 100644 index 00000000..bc3c5035 --- /dev/null +++ b/repo/packages/go/package.py.old @@ -0,0 +1,168 @@ +# just added a version +########################################## +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os +import re + +import llnl.util.tty as tty + +from spack.package import * + +# - vanilla CentOS 7, and possibly other systems, fail a test: +# TestCloneNEWUSERAndRemapRootDisableSetgroups +# +# The Fix, discussed here: https://github.com/golang/go/issues/16283 +# is to enable "user_namespace". +# +# On a Digital Ocean image, this can be achieved by updating +# `/etc/default/grub` so that the `GRUB_CMDLINE_LINUX` variable +# includes `user_namespace.enable=1`, re-cooking the grub +# configuration with `sudo grub2-mkconfig -o /boot/grub2/grub.cfg`, +# and then rebooting. +# +# - on CentOS 7 systems (and possibly others) you need to have the +# glibc package installed or various static cgo tests fail. +# + + +class Go(Package): + """The golang compiler and build environment""" + + homepage = "https://go.dev" + url = "https://go.dev/dl/go1.20.2.src.tar.gz" + git = "https://go.googlesource.com/go.git" + + extendable = True + executables = ["^go$"] + + maintainers = ["alecbcs"] + + version("1.20.4", sha256="9f34ace128764b7a3a4b238b805856cc1b2184304df9e5690825b0710f4202d6") + version("1.20.3", sha256="e447b498cde50215c4f7619e5124b0fc4e25fb5d16ea47271c47f278e7aa763a") + + version("1.19.10", sha256="13755bcce529747d5f2930dee034730c86d02bd3e521ab3e2bbede548d3b953f") + version("1.19.9", sha256="131190a4697a70c5b1d232df5d3f55a3f9ec0e78e40516196ffb3f09ae6a5744") + version("1.19.8", sha256="1d7a67929dccafeaf8a29e55985bc2b789e0499cb1a17100039f084e3238da2f") + + # Deprecated Versions + # https://nvd.nist.gov/vuln/detail/CVE-2023-24538 + version( + "1.20.2", + sha256="4d0e2850d197b4ddad3bdb0196300179d095bb3aefd4dfbc3b36702c3728f8ab", + deprecated=True, + ) + version( + "1.19.7", + sha256="775bdf285ceaba940da8a2fe20122500efd7a0b65dbcee85247854a8d7402633", + deprecated=True, + ) + + # https://nvd.nist.gov/vuln/detail/CVE-2023-24532 + version( + "1.20.1", + sha256="b5c1a3af52c385a6d1c76aed5361cf26459023980d0320de7658bae3915831a2", + deprecated=True, + ) + # https://nvd.nist.gov/vuln/detail/CVE-2022-41723 + version( + "1.20", + sha256="3a29ff0421beaf6329292b8a46311c9fbf06c800077ceddef5fb7f8d5b1ace33", + deprecated=True, + ) + # https://nvd.nist.gov/vuln/detail/CVE-2022-41725 + version( + "1.19.6", + sha256="d7f0013f82e6d7f862cc6cb5c8cdb48eef5f2e239b35baa97e2f1a7466043767", + deprecated=True, + ) + # https://nvd.nist.gov/vuln/detail/CVE-2022-41725 + version( + "1.19.5", + sha256="8e486e8e85a281fc5ce3f0bedc5b9d2dbf6276d7db0b25d3ec034f313da0375f", + deprecated=True, + ) + version( + "1.19.4", + sha256="eda74db4ac494800a3e66ee784e495bfbb9b8e535df924a8b01b1a8028b7f368", + deprecated=True, + ) + # https://nvd.nist.gov/vuln/detail/CVE-2022-41724 + version( + "1.18.10", + sha256="9cedcca58845df0c9474ae00274c44a95c9dfaefb132fc59921c28c7c106f8e6", + deprecated=True, + ) + version( + "1.18.9", + sha256="fbe7f09b96aca3db6faeaf180da8bb632868ec049731e355ff61695197c0e3ea", + deprecated=True, + ) + + provides("golang") + + depends_on("git", type=("build", "link", "run")) + depends_on("go-or-gccgo-bootstrap", type="build") + depends_on("go-or-gccgo-bootstrap@1.17.13:", type="build", when="@1.20:") + + phases = ["build", "install"] + + def url_for_version(self, version): + return f"https://go.dev/dl/go{version}.src.tar.gz" + + @classmethod + def determine_version(cls, exe): + output = Executable(exe)("version", output=str, error=str) + match = re.search(r"go version go(\S+)", output) + return match.group(1) if match else None + + def setup_build_environment(self, env): + env.set("GOROOT_FINAL", self.spec.prefix) + # We need to set CC/CXX_FOR_TARGET, otherwise cgo will use the + # internal Spack wrappers and fail. + env.set("CC_FOR_TARGET", self.compiler.cc) + env.set("CXX_FOR_TARGET", self.compiler.cxx) + env.set("GOMAXPROCS", make_jobs) + + def build(self, spec, prefix): + bash = which("bash") + + with working_dir("src"): + bash("{0}.bash".format("all" if self.run_tests else "make")) + + def install(self, spec, prefix): + install_tree(".", prefix) + + def setup_dependent_package(self, module, dependent_spec): + """Called before go modules' build(), install() methods. + + In most cases, extensions will only need to set GOPATH and use go:: + + env['GOPATH'] = self.source_path + ':' + env['GOPATH'] + go('get', '', env=env) + install_tree('bin', prefix.bin) + """ + # Add a go command/compiler for extensions + module.go = self.spec["go"].command + + def generate_path_components(self, dependent_spec): + if os.environ.get("GOROOT", False): + tty.warn("GOROOT is set, this is not recommended") + + # Set to include paths of dependencies + path_components = [dependent_spec.prefix] + for d in dependent_spec.traverse(): + if d.package.extends(self.spec): + path_components.append(d.prefix) + return ":".join(path_components) + + def setup_dependent_build_environment(self, env, dependent_spec): + # This *MUST* be first, this is where new code is installed + env.prepend_path("GOPATH", self.generate_path_components(dependent_spec)) + + def setup_dependent_run_environment(self, env, dependent_spec): + # Allow packages to find this when using module files + env.prepend_path("GOPATH", self.generate_path_components(dependent_spec)) diff --git a/repo/packages/go/time_test.patch b/repo/packages/go/time_test.patch new file mode 100644 index 00000000..c3e0697c --- /dev/null +++ b/repo/packages/go/time_test.patch @@ -0,0 +1,18 @@ +diff --git a/src/time/time_test.go b/src/time/time_test.go +index 68236fd..2e47d08 100644 +--- a/src/time/time_test.go ++++ b/src/time/time_test.go +@@ -943,8 +943,11 @@ func TestLoadFixed(t *testing.T) { + // but Go and most other systems use "east is positive". + // So GMT+1 corresponds to -3600 in the Go zone, not +3600. + name, offset := Now().In(loc).Zone() +- if name != "GMT+1" || offset != -1*60*60 { +- t.Errorf("Now().In(loc).Zone() = %q, %d, want %q, %d", name, offset, "GMT+1", -1*60*60) ++ // The zone abbreviation is "-01" since tzdata-2016g, and "GMT+1" ++ // on earlier versions; we accept both. (Issue #17276). ++ if !(name == "GMT+1" || name == "-01") || offset != -1*60*60 { ++ t.Errorf("Now().In(loc).Zone() = %q, %d, want %q or %q, %d", ++ name, offset, "GMT+1", "-01", -1*60*60) + } + } + diff --git a/repo/packages/singularityce/package.py b/repo/packages/singularityce/package.py new file mode 100644 index 00000000..62506f1e --- /dev/null +++ b/repo/packages/singularityce/package.py @@ -0,0 +1,212 @@ +#updated dependency to conmon and new versions +#+ variant("conmon", default=False, description="required to run OCI container monitor") +#+ depends_on("conmon", when="+conmon") +############################################################################## +# Copyright 2013-2023 Lawrence Livermore National Security, LLC and other +# Spack Project Developers. See the top-level COPYRIGHT file for details. +# +# SPDX-License-Identifier: (Apache-2.0 OR MIT) + +import os +import shutil + +import llnl.util.tty as tty + +from spack.package import * + + +class SingularityBase(MakefilePackage): + variant("suid", default=True, description="install SUID binary") + variant("network", default=True, description="install network plugins") + variant("conmon", default=False, description="required to run OCI container monitor") + + depends_on("pkgconfig", type="build") + depends_on("conmon", type=("build", "run")) + depends_on("squashfs", type=("build", "run")) + depends_on("go@1.16:") + depends_on("uuid") + depends_on("libgpg-error") + depends_on("libseccomp") + depends_on("squashfs", type="run") + depends_on("git", when="@develop") # mconfig uses it for version info + depends_on("shadow", type="run", when="@3.3:") + depends_on("cryptsetup", type=("build", "run"), when="@3.4:") + depends_on("conmon", when="+conmon") + + conflicts("platform=darwin", msg="singularity requires a Linux VM on Windows & Mac") + + # Use these properties to buffer the renaming to Apptainer + singularity_org = "sylabs" + singularity_name = "singularity" + singularity_security_urls = ( + "https://sylabs.io/guides/2.6/admin-guide/security.html", + "https://sylabs.io/guides/3.2/admin-guide/admin_quickstart.html#singularity-security", + ) + + # Go has novel ideas about how projects should be organized. + # We'll point GOPATH at the stage dir, and move the unpacked src + # tree into the proper subdir in our overridden do_stage below. + @property + def gopath(self): + return self.stage.path + + @property + def singularity_gopath_dir(self): + return join_path( + self.gopath, "src", "github.com", self.singularity_org, self.singularity_name + ) + + # Unpack the tarball as usual, then move the src dir into + # its home within GOPATH. + def do_stage(self, mirror_only=False): + super(SingularityBase, self).do_stage(mirror_only) + if not os.path.exists(self.singularity_gopath_dir): + # Move the expanded source to its destination + tty.debug( + "Moving {0} to {1}".format(self.stage.source_path, self.singularity_gopath_dir) + ) + shutil.move(self.stage.source_path, self.singularity_gopath_dir) + + # The build process still needs access to the source path, + # so create a symlink. + force_symlink(self.singularity_gopath_dir, self.stage.source_path) + + # MakefilePackage's stages use this via working_dir() + @property + def build_directory(self): + return self.singularity_gopath_dir + + # Hijack the edit stage to run mconfig. + def edit(self, spec, prefix): + with working_dir(self.build_directory): + confstring = "./mconfig --prefix=%s" % prefix + if "~suid" in spec: + confstring += " --without-suid" + if "~network" in spec: + confstring += " --without-network" + if "~conmon" in spec: + confstring += " --without-conmon" + configure = Executable(confstring) + configure() + + # Set these for use by MakefilePackage's default build/install methods. + build_targets = ["-C", "builddir", "parallel=False"] + install_targets = ["install", "-C", "builddir", "parallel=False"] + + def setup_build_environment(self, env): + # Point GOPATH at the top of the staging dir for the build step. + env.prepend_path("GOPATH", self.gopath) + + # `singularity` has a fixed path where it will look for + # mksquashfs. If it lives somewhere else you need to specify the + # full path in the config file. This bit uses filter_file to edit + # the config file, uncommenting and setting the mksquashfs path. + @run_after("install") + def fix_mksquashfs_path(self): + prefix = self.spec.prefix + squash_path = join_path(self.spec["squashfs"].prefix.bin, "mksquashfs") + filter_file( + r"^# mksquashfs path =", + "mksquashfs path = {0}".format(squash_path), + join_path(prefix.etc, self.singularity_name, self.singularity_name + ".conf"), + ) + + # + # Assemble a script that fixes the ownership and permissions of several + # key files, install it, and tty.warn() the user. + # HEADSUP: https://github.com/spack/spack/pull/10412. + # + def perm_script(self): + return "spack_perms_fix.sh" + + def perm_script_tmpl(self): + return "{0}.j2".format(self.perm_script()) + + def perm_script_path(self): + return join_path(self.spec.prefix.bin, self.perm_script()) + + def _build_script(self, filename, variable_data): + with open(filename, "w") as f: + env = spack.tengine.make_environment(dirs=self.package_dir) + t = env.get_template(self.perm_script_tmpl()) + f.write(t.render(variable_data)) + + @run_after("install") + def build_perms_script(self): + if self.spec.satisfies("+suid"): + script = self.perm_script_path() + chown_files = [ + fn.format(self.singularity_name) + for fn in [ + "libexec/{0}/bin/starter-suid", + "etc/{0}/{0}.conf", + "etc/{0}/capability.json", + "etc/{0}/ecl.toml", + ] + ] + setuid_files = ["libexec/{0}/bin/starter-suid".format(self.singularity_name)] + self._build_script( + script, + { + "prefix": self.spec.prefix, + "chown_files": chown_files, + "setuid_files": setuid_files, + }, + ) + chmod = which("chmod") + chmod("555", script) + + # Until tty output works better from build steps, this ends up in + # the build log. See https://github.com/spack/spack/pull/10412. + @run_after("install") + def caveats(self): + if self.spec.satisfies("+suid"): + tty.warn( + """ + For full functionality, you'll need to chown and chmod some files + after installing the package. This has security implications. + For details, see: + {1} + {2} + + We've installed a script that will make the necessary changes; + read through it and then execute it as root (e.g. via sudo). + + The script is named: + + {0} + """.format( + self.perm_script_path(), *self.singularity_security_urls + ) + ) + + +class Singularityce(SingularityBase): + """Singularity is a container technology focused on building portable + encapsulated environments to support "Mobility of Compute" For older + versions of Singularity (pre 3.0) you should use singularity-legacy, + which has a different install base (Autotools). + + Needs post-install chmod/chown steps to enable full functionality. + See package definition or `spack-build-out.txt` build log for details, + e.g. + + tail -15 $(spack location -i singularity)/.spack/spack-build-out.txt + """ + + homepage = "https://sylabs.io/singularity/" + url = "https://github.com/sylabs/singularity/releases/download/v3.9.1/singularity-ce-3.9.1.tar.gz" + git = "https://github.com/sylabs/singularity.git" + + maintainers = ['alalazo'] + version("master", branch="master") + + version("3.11.4", sha256="751dbea64ec16fd7e7af1e36953134c778c404909f9d27ba89006644160b2fde") + version("3.11.3", sha256="a77ede063fd115f85f98f82d2e30459b5565db7d098665497bcd684bf8edaec9") + version("3.10.3", sha256="f87d8e212ce209c5212d6faf253b97a24b5d0b6e6b17b5e58b316cdda27a332f") + version("3.10.2", sha256="b4f279856ea4bf28a1f34f89320c02b545d6e57d4143679920e1ac4267f540e1") + version("3.10.1", sha256="e3af12edc0260bc3a3a481459a3a4457de9235025e6b37288da80e3cdc011a7a") + version("3.10.0", sha256="5e22e6cdad66c331668f6cff4544c83917bb3db90da3cf92403a394c5bf8cc8f") + version("3.9.9", sha256="1381433d64138c08e93ffacdfb4844e82c2288f1e39a9d2c631a1c4021381f2a") + version("3.9.1", sha256="1ba3bb1719a420f48e9b0a6afdb5011f6c786d0f107ef272528c632fff9fd153") + version("3.8.0", sha256="5fa2c0e7ef2b814d8aa170826b833f91e5031a85d85cd1292a234e6c55da1be1") diff --git a/repo/packages/singularityce/spack_perms_fix.sh.j2 b/repo/packages/singularityce/spack_perms_fix.sh.j2 new file mode 100644 index 00000000..32baa212 --- /dev/null +++ b/repo/packages/singularityce/spack_perms_fix.sh.j2 @@ -0,0 +1,11 @@ +#!/bin/sh -eu + +{% for cf in chown_files %} +chown root {{ prefix }}/{{ cf }} +{% endfor %} + +{% for sf in setuid_files %} +chmod 4555 {{ prefix }}/{{ sf }} +{% endfor %} + +# end diff --git a/scripts/create_custom_singularity_modules.sh b/scripts/create_custom_singularity_modules.sh index dda6b2ed..f4e693a8 100755 --- a/scripts/create_custom_singularity_modules.sh +++ b/scripts/create_custom_singularity_modules.sh @@ -17,15 +17,14 @@ PAWSEY_SPACK_CONFIG_REPO=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )/.." &> /d # This script creates end user singularity modules, starting from the Spack one # For each version: -# 1. creates 3 pawsey/modules : -# A. singularity-nompi -# B. singularity-mpi -# C. singularity-astro -# The purpose are: -# - to have Singularity available regardless the loaded PrgEng/compiler -# (works thanks to RPATHs) -# - to have /astro and /askapbuffer only for astronomers -# - to have MPI bind mounts only when needed +# 1. creates several modules each setting different default behaviour: +# A.1 singularity/x.x.x-mpi : injects all necessary libraries to run MPI, binds filesystems /scratch, /software +# A.2 singularity/x.x.x-mpi-gpu : injects all necessary libraries to run MPI, binds filesystems /scratch, /software, etc and also preloads the gtl library and sets the MPICH variable to enable GPU-GPU MPI. +# B. singularity/x.x.x-nompi : binds filesystems, also sets LD path based on currenty LD path +# C.1 singularity/x.x.x-askap : like MPI but also addes /askapbuffer and also adds slurm related bind mounts +# C.2 singularity/x.x.x-askap-gpu : like MPI but also addes /askapbuffer and also adds slurm related bind mounts +# D. singularity/x.x.x-nohost : nothing from the host is injected, does mount file systems +# E. singularity/x.x.x-slurm : like nompi but also mounts directories related to running slurm in a container # 2. hides Spack module in {spack modules}/{arch}/{compiler} # - to avoid confusion @@ -38,28 +37,57 @@ mkdir -p ${dst_dir} # remove old pawsey singularity modules rm -f ${dst_dir}/*.lua -for version in $( ls ${src_dir}/*.lua ${src_dir}/.*.lua 2>/dev/null ) ; do +# older script also processed .*.lua but shouldn't +#for version in $( ls ${src_dir}/*.lua ${src_dir}/.*.lua 2>/dev/null ) ; do +for version in $( ls ${src_dir}/*.lua 2>/dev/null ) ; do version="${version##*/}" - version="${version#.}" + #version="${version#.lua}" + version=$(echo ${version} | awk -F.lua '{print $1}') + echo "Generating specific modules for singularity ${version} " #2. hide Spack module - if [ -e ${src_dir}/${version} ] ; then - mv ${src_dir}/${version} ${src_dir}/.${version} + if [ -e ${src_dir}/${version}.lua ] ; then + mv ${src_dir}/${version}.lua ${src_dir}/.${version} fi - # 1.C singularity-astro is just the original module - cp -p ${src_dir}/.${version} ${dst_dir}/${version/\.lua/-astro.lua} - # 1.B singularity-mpi does not bind mount /askapbuffer and /astro + # 1.C.2 singularity-askap is just the original module + cp -p ${src_dir}/.${version} ${dst_dir}/${version}-askap-gpu.lua + # 1.C.1 singularity-askap is just the original module + sed \ + -e '/^-- add GPUMPI START/,/^-- add GPUMPI END/{/^-- add GPUMPI START/!{/^-- add GPUMPI END/!d}}' \ + ${src_dir}/.${version} > ${dst_dir}/${version}-askap.lua + # 1.A.2 singularity does not bind mount /askapbuffer nor add slurm + # the sed command removes some entries and also deletes items between specific lines + # leaving the delimiters between the lines intact for easy checks. + sed \ + -e '/singularity_bindpath *=/ s;/askapbuffer;;g' \ + -e '/^-- add SLURM START/,/^-- add SLURM END/{/^-- add SLURM START/!{/^-- add SLURM END/!d}}' \ + ${src_dir}/.${version} > ${dst_dir}/${version}-mpi-gpu.lua + # 1.A.1 + sed \ + -e '/singularity_bindpath *=/ s;/askapbuffer;;g' \ + -e '/^-- add SLURM START/,/^-- add SLURM END/{/^-- add SLURM START/!{/^-- add SLURM END/!d}}' \ + -e '/^-- add GPUMPI START/,/^-- add GPUMPI END/{/^-- add GPUMPI START/!{/^-- add GPUMPI END/!d}}' \ + ${src_dir}/.${version} > ${dst_dir}/${version}-mpi.lua + # 1.B singularity does not add any mpi related stuff + sed \ + -e '/singularity_bindpath *=/ s;/askapbuffer;;g' \ + -e '/^-- add MPI START/,/^-- add MPI END/{/^-- add MPI START/!{/^-- add MPI END/!d}}' \ + -e '/^-- add SLURM START/,/^-- add SLURM END/{/^-- add SLURM START/!{/^-- add SLURM END/!d}}' \ + -e '/^-- add GPUMPI START/,/^-- add GPUMPI END/{/^-- add GPUMPI START/!{/^-- add GPUMPI END/!d}}' \ + ${src_dir}/.${version} > ${dst_dir}/${version}-nompi.lua + # 1.D singularity does not add any thing at all from host sed \ -e '/singularity_bindpath *=/ s;/askapbuffer;;g' \ - -e '/singularity_bindpath *=/ s;/astro;;g' \ - ${dst_dir}/${version/\.lua/-astro.lua} \ - >${dst_dir}/${version/\.lua/-mpi.lua} - # 1.A singularity-nompi does not bind mount MPI-specific paths + -e '/^-- add MPI START/,/^-- add MPI END/{/^-- add MPI START/!{/^ add MPI END/!d}}' \ + -e '/^-- add SLURM START/,/^-- add SLURM END/{/^-- add SLURM START/!{/^-- add SLURM END/!d}}' \ + -e '/^-- add GPUMPI START/,/^-- add GPUMPI END/{/^-- add GPUMPI START/!{/^-- add GPUMPI END/!d}}' \ + -e '/^-- add CRAY_PATHS START/,/^-- add CRAY_PATHS END/{/^-- add CRAY_PATHS START/!{/^-- add CRAY_PATHS END/!d}}' \ + -e '/^-- add CURRENT_HOST_LD_PATH START/,/^-- add CURRENT_HOST_LD_PATH END/{/^-- add CURRENT_HOST_LD_PATH START/!{/^-- add CURRENT_HOST_LD_PATH END/!d}}' \ + ${src_dir}/.${version} > ${dst_dir}/${version}-nohost.lua + # 1.B singularity does not add any mpi related stuff but keeps slurm related mounts sed \ - -e '/MPI + Singularity configuration/,/singularity_ld_path *= *singularity_ld_path/d' \ - -e 's;, *singularity_ld_path;,"$LD_LIBRARY_PATH";' \ - -e 's;,/etc/alternatives/cray-xpmem;;' \ - -e '/singularity_bindpath.*libselinux/,/singularity_bindpath.*libmunge/d' \ - -e '/LD_PRELOAD edited/,/, *singularity_ld_preload/d' \ - ${dst_dir}/${version/\.lua/-mpi.lua} \ - >${dst_dir}/${version/\.lua/-nompi.lua} + -e '/singularity_bindpath *=/ s;/askapbuffer;;g' \ + -e '/^-- add MPI START/,/^-- add MPI END/{/^-- add MPI START/!{/^-- add MPI END/!d}}' \ + -e '/^-- add GPUMPI START/,/^-- add GPUMPI END/{/^-- add GPUMPI START/!{/^-- add GPUMPI END/!d}}' \ + ${src_dir}/.${version} > ${dst_dir}/${version}-slurm.lua + echo "Finished generating modules for ${version}" done diff --git a/scripts/create_custom_singularity_modules_from_general_singularity_container_engine.sh b/scripts/create_custom_singularity_modules_from_general_singularity_container_engine.sh new file mode 100755 index 00000000..84f63523 --- /dev/null +++ b/scripts/create_custom_singularity_modules_from_general_singularity_container_engine.sh @@ -0,0 +1,107 @@ +#!/bin/bash -e + +# This script generates singularity named modules +# from the singularityce spack module +# it is effectively the same as create_custom_singularity_modules_from_singularity.sh +# the main difference is that the src_dr is singularityce_name and destination +# uses singularity_name_general as the general container engine +# allowing the use of singularityce or apptainer recipes. + +# source setup variables +# if copy/pasting these commands, need to run from this directory +if [ -n "${PAWSEY_CLUSTER}" ] && [ -z ${SYSTEM+x} ]; then + SYSTEM="$PAWSEY_CLUSTER" +fi + +if [ -z ${SYSTEM+x} ]; then + echo "The 'SYSTEM' variable is not set. Please specify the system you want to + build Spack for." + exit 1 +fi + +PAWSEY_SPACK_CONFIG_REPO=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )/.." &> /dev/null && pwd ) +. "${PAWSEY_SPACK_CONFIG_REPO}/systems/${SYSTEM}/settings.sh" + +# This script creates end user singularity modules, starting from the Spack one +# For each version: +# 1. creates several modules each setting different default behaviour: +# A.1 singularity/x.x.x-mpi : injects all necessary libraries to run MPI, binds filesystems /scratch, /software +# A.2 singularity/x.x.x-mpi-gpu : injects all necessary libraries to run MPI, binds filesystems /scratch, /software, etc and also preloads the gtl library and sets the MPICH variable to enable GPU-GPU MPI. +# B. singularity/x.x.x-nompi : binds filesystems, also sets LD path based on currenty LD path +# C.1 singularity/x.x.x-askap : like MPI but also addes /askapbuffer and also adds slurm related bind mounts +# C.2 singularity/x.x.x-askap-gpu : like MPI but also addes /askapbuffer and also adds slurm related bind mounts +# D. singularity/x.x.x-nohost : nothing from the host is injected, does mount file systems +# E. singularity/x.x.x-slurm : like nompi but also mounts directories related to running slurm in a container +# 2. hides Spack module in {spack modules}/{arch}/{compiler} +# - to avoid confusion + +# define source (Spack) and destination (Pawsey) directories for Singularity +src_old_dir="${INSTALL_PREFIX}/modules/${cpu_arch}/gcc/${gcc_version}/utilities/${singularity_name_general}" +src_dir="${INSTALL_PREFIX}/modules/${cpu_arch}/gcc/${gcc_version}/utilities/${singularity_name}" +dst_dir="${INSTALL_PREFIX}/${singularity_symlink_module_dir}" +echo "Using module files in ${src_old_dir}" +echo "Copying them to ${src_dir}" +echo "And using this basis to generate modules in ${dst_dir}" + +# copy the singularityce modules to singularity directory +cp ${src_old_dir}/*.lua ${src_dir}/ + +# ensure destination directory exists +mkdir -p ${dst_dir} +# remove old pawsey singularity modules +rm -f ${dst_dir}/*.lua + +# older script also processed .*.lua but shouldn't +#for version in $( ls ${src_dir}/*.lua ${src_dir}/.*.lua 2>/dev/null ) ; do +for version in $( ls ${src_dir}/*.lua 2>/dev/null ) ; do + version="${version##*/}" + #version="${version#.lua}" + version=$(echo ${version} | awk -F.lua '{print $1}') + echo "Generating specific modules for singularity ${version} " + #2. hide Spack module + if [ -e ${src_dir}/${version}.lua ] ; then + mv ${src_dir}/${version}.lua ${src_dir}/.${version} + fi + # 1.C.2 singularity-askap is just the original module + cp -p ${src_dir}/.${version} ${dst_dir}/${version}-askap-gpu.lua + # 1.C.1 singularity-askap is just the original module + sed \ + -e '/^-- add GPUMPI START/,/^-- add GPUMPI END/{/^-- add GPUMPI START/!{/^-- add GPUMPI END/!d}}' \ + ${src_dir}/.${version} > ${dst_dir}/${version}-askap.lua + # 1.A.2 singularity does not bind mount /askapbuffer nor add slurm + # the sed command removes some entries and also deletes items between specific lines + # leaving the delimiters between the lines intact for easy checks. + sed \ + -e '/singularity_bindpath *=/ s;/askapbuffer;;g' \ + -e '/^-- add SLURM START/,/^-- add SLURM END/{/^-- add SLURM START/!{/^-- add SLURM END/!d}}' \ + ${src_dir}/.${version} > ${dst_dir}/${version}-mpi-gpu.lua + # 1.A.1 + sed \ + -e '/singularity_bindpath *=/ s;/askapbuffer;;g' \ + -e '/^-- add SLURM START/,/^-- add SLURM END/{/^-- add SLURM START/!{/^-- add SLURM END/!d}}' \ + -e '/^-- add GPUMPI START/,/^-- add GPUMPI END/{/^-- add GPUMPI START/!{/^-- add GPUMPI END/!d}}' \ + ${src_dir}/.${version} > ${dst_dir}/${version}-mpi.lua + # 1.B singularity does not add any mpi related stuff + sed \ + -e '/singularity_bindpath *=/ s;/askapbuffer;;g' \ + -e '/^-- add MPI START/,/^-- add MPI END/{/^-- add MPI START/!{/^-- add MPI END/!d}}' \ + -e '/^-- add SLURM START/,/^-- add SLURM END/{/^-- add SLURM START/!{/^-- add SLURM END/!d}}' \ + -e '/^-- add GPUMPI START/,/^-- add GPUMPI END/{/^-- add GPUMPI START/!{/^-- add GPUMPI END/!d}}' \ + ${src_dir}/.${version} > ${dst_dir}/${version}-nompi.lua + # 1.D singularity does not add any thing at all from host + sed \ + -e '/singularity_bindpath *=/ s;/askapbuffer;;g' \ + -e '/^-- add MPI START/,/^-- add MPI END/{/^-- add MPI START/!{/^ add MPI END/!d}}' \ + -e '/^-- add SLURM START/,/^-- add SLURM END/{/^-- add SLURM START/!{/^-- add SLURM END/!d}}' \ + -e '/^-- add GPUMPI START/,/^-- add GPUMPI END/{/^-- add GPUMPI START/!{/^-- add GPUMPI END/!d}}' \ + -e '/^-- add CRAY_PATHS START/,/^-- add CRAY_PATHS END/{/^-- add CRAY_PATHS START/!{/^-- add CRAY_PATHS END/!d}}' \ + -e '/^-- add CURRENT_HOST_LD_PATH START/,/^-- add CURRENT_HOST_LD_PATH END/{/^-- add CURRENT_HOST_LD_PATH START/!{/^-- add CURRENT_HOST_LD_PATH END/!d}}' \ + ${src_dir}/.${version} > ${dst_dir}/${version}-nohost.lua + # 1.B singularity does not add any mpi related stuff but keeps slurm related mounts + sed \ + -e '/singularity_bindpath *=/ s;/askapbuffer;;g' \ + -e '/^-- add MPI START/,/^-- add MPI END/{/^-- add MPI START/!{/^-- add MPI END/!d}}' \ + -e '/^-- add GPUMPI START/,/^-- add GPUMPI END/{/^-- add GPUMPI START/!{/^-- add GPUMPI END/!d}}' \ + ${src_dir}/.${version} > ${dst_dir}/${version}-slurm.lua + echo "Finished generating modules for ${version}" +done diff --git a/scripts/install_software_stack.sh b/scripts/install_software_stack.sh index 51d472e7..ede487e7 100755 --- a/scripts/install_software_stack.sh +++ b/scripts/install_software_stack.sh @@ -26,7 +26,9 @@ echo "Install environments.." "${PAWSEY_SPACK_CONFIG_REPO}/scripts/install_environments.sh" echo "Update singularity modules.." -"${PAWSEY_SPACK_CONFIG_REPO}/scripts/create_custom_singularity_modules.sh" +# note that moving forward using the singularityce recipes. Comment out the singularity_modules, kept for reference +#"${PAWSEY_SPACK_CONFIG_REPO}/scripts/create_custom_singularity_modules.sh" +"${PAWSEY_SPACK_CONFIG_REPO}/scripts/create_custom_singularity_modules_from_general_singularity_container_engine.sh" echo "Installing shpc..." "${PAWSEY_SPACK_CONFIG_REPO}/scripts/install_shpc.sh" diff --git a/systems/setonix/configs/site/packages.yaml b/systems/setonix/configs/site/packages.yaml index 412cdbf2..1e7bf910 100644 --- a/systems/setonix/configs/site/packages.yaml +++ b/systems/setonix/configs/site/packages.yaml @@ -132,7 +132,7 @@ packages: version: [68.0.0] # preferred versions and variants, compilers go: - version: [1.17.2] + version: [1.19.10] julia: version: [1.8.2] openjdk: diff --git a/systems/setonix/environments/env_langs/spack.yaml b/systems/setonix/environments/env_langs/spack.yaml index 9ed9c845..7756ae86 100644 --- a/systems/setonix/environments/env_langs/spack.yaml +++ b/systems/setonix/environments/env_langs/spack.yaml @@ -4,7 +4,11 @@ spack: # aocc, rocm, gcc, llvm: provided by system installations - perl@5.36.0 - - go@1.17.2 + # this is a proposed change to eliminate openss and older go + #- openssl@3.0.7 ^perl@5.36.0 + # do we really need openssl 3.0.7? + #- go@1.17.2 ^openssl@3.0.7 + - go@1.19.10 # NotReadyYet: # - julia@1.6.1 #Default recipe is very weak. Faulty. Lots of needed dependencies are missing. diff --git a/systems/setonix/environments/env_utils/spack.yaml b/systems/setonix/environments/env_utils/spack.yaml index ea293a0e..855daf04 100644 --- a/systems/setonix/environments/env_utils/spack.yaml +++ b/systems/setonix/environments/env_utils/spack.yaml @@ -12,7 +12,8 @@ spack: - parallel@20220522 - reframe@3.12.0 ^python@3.10.10 +optimizations+zlib - reframe@3.10.1 ^python@3.10.10 +optimizations+zlib - - singularity@3.8.7 + # now moving to singularityce rather than older singularity + - singularityce@3.11.4~conmon ^go@1.19.10 - mpifileutils@0.11.1+lustre - feh@3.3 # utilities for Nextflow Tower diff --git a/systems/setonix/settings.sh b/systems/setonix/settings.sh index baf0a6b5..629f4569 100644 --- a/systems/setonix/settings.sh +++ b/systems/setonix/settings.sh @@ -52,8 +52,8 @@ cpu_arch="zen3" # tool versions spack_version="0.19.0" # the prefix "v" is added in setup_spack.sh -singularity_version="3.8.7-nompi" # has to match the version in the Spack env yaml + nompi tag -singularity_mpi_version="3.8.7-mpi" # has to match the version in the Spack env yaml + mpi tag +singularity_version="3.11.4-nompi" # has to match the version in the Spack env yaml + nompi tag +singularity_mpi_version="3.11.4-mpi" # has to match the version in the Spack env yaml + mpi tag shpc_version="0.1.22" shpc_registry_version="61a74214fb6d3b7667103cbaa022746fa931438d" @@ -156,6 +156,7 @@ shpc_name="shpc" # name of Singularity module (Spack has singularity and singularityce) singularity_name="singularity" +singularity_name_general="singularityce" # NOTE: the following are ALL RELATIVE to root_dir above # root location for Pawsey custom builds diff --git a/systems/setonix/templates/modules/modulefile.lua b/systems/setonix/templates/modules/modulefile.lua index b3679662..bea5743a 100644 --- a/systems/setonix/templates/modules/modulefile.lua +++ b/systems/setonix/templates/modules/modulefile.lua @@ -94,22 +94,86 @@ setenv("NXF_SINGULARITY_CACHEDIR", os.getenv("MYSOFTWARE").."/.nextflow_singular {% endif %} {% if spec.name == 'openjdk' %}setenv("GRADLE_USER_HOME", os.getenv("MYSOFTWARE").."/.gradle") {% endif %} -{% if spec.name == 'singularity' %}setenv("SINGULARITY_CACHEDIR", os.getenv("MYSOFTWARE").."/.singularity") --- MPI + Singularity configuration -setenv("SINGULARITYENV_LD_LIBRARY_PATH","/opt/cray/pe/mpich/default/ofi/gnu/9.1/lib-abi-mpich:/opt/cray/pe/mpich/default/gtl/lib:/opt/cray/xpmem/default/lib64:/opt/cray/pe/pmi/default/lib:/opt/cray/pe/pals/default/lib:/opt/cray/libfabric/1.11.0.4.128/lib64:/opt/cray/pe/gcc-libs:/usr/lib64/libibverbs:$LD_LIBRARY_PATH") --- bind paths of standard filesystems and specific libraries -local singularity_bindpath = "/askapbuffer,/astro,/scratch,/software,/var/opt/cray/pe,/etc/opt/cray/pe,/opt/cray,/etc/alternatives/cray-dvs,/etc/alternatives/cray-xpmem" -singularity_bindpath = singularity_bindpath .. ",/usr/lib64/librdmacm.so.1,/usr/lib64/libibverbs.so.1,/usr/lib64/libnl-3.so.200,/usr/lib64/libnl-route-3.so.200,/usr/lib64/libmlx5.so.1,/usr/lib64/libmunge.so.2,/etc/libibverbs.d,/usr/lib64/libibverbs" +{% if spec.name == 'singularity' or spec.name == 'singularityce' %}setenv("SINGULARITY_CACHEDIR", os.getenv("MYSOFTWARE").."/.singularity") +-- Singularity configuration START +-- LD_LIBRARY_PATH addition +local singularity_ld_path = "" +-- add CRAY_PATHS START +singularity_ld_path = singularity_ld_path .. ":/opt/cray/pe/mpich/default/ofi/gnu/9.1/lib-abi-mpich:/opt/cray/pe/mpich/default/gtl/lib:/opt/cray/xpmem/default/lib64:/opt/cray/pe/pmi/default/lib:/opt/cray/pe/pals/default/lib" +singularity_ld_path = singularity_ld_path .. ":/opt/cray/pe/gcc-libs" +-- add CRAY_PATHS END +-- add MPI START +-- for Cassini nics and SS>=11 +-- add libfabric, might need version changes +singularity_ld_path = singularity_ld_path .. ":/opt/cray/libfabric/1.15.2.0/lib64/" +-- add MPI END +-- add CURRENT_HOST_LD_PATH START +singularity_ld_path = singularity_ld_path .. ":$LD_LIBRARY_PATH" +-- add CURRENT_HOST_LD_PATH END +setenv("SINGULARITYENV_LD_LIBRARY_PATH", singularity_ld_path) + +-- BIND_PATH addition +-- add LOCAL FILESYSTEM START +local singularity_bindpath = "/askapbuffer,/scratch,/software" +-- add LOCAL FILESYSTEM END +-- add SLURM START +singularity_bindpath = singularity_bindpath .. ",/var/run/munge/munge.socket.2,/etc/slurm" +-- add SLURM END +-- add CRAY_PATHS START +singularity_bindpath = singularity_bindpath .. ",/var/opt/cray/pe,/etc/opt/cray/pe,/opt/cray,/etc/alternatives/cray-dvs,/etc/alternatives/cray-xpmem" +-- commented out below, adding lib64. Could be useful +-- singularity_bindpath = singularity_bindpath .. ",/lib64/libc.so.6,/lib64/libpthread.so.0,/lib64/librt.so.1,/lib64/libdl.so.2,/lib64/libz.so.1,/lib64/libselinux.so.1,/lib64/libm.so.6" +-- add CRAY_PATHS END +-- add MPI START +-- for Cassini nics and SS>=11 +-- added several different libraries to ensure that mpi works +singularity_bindpath = singularity_bindpath .. ",/usr/lib64/libcxi.so.1,/usr/lib64/libcurl.so.4,/usr/lib64/libjson-c.so.3" +singularity_bindpath = singularity_bindpath .. ",/usr/lib64/libnghttp2.so.14,/usr/lib64/libidn2.so.0,/usr/lib64/libssh.so.4,/usr/lib64/libpsl.so.5,/usr/lib64/libssl.so.1.1,/usr/lib64/libcrypto.so.1.1,/usr/lib64/libgssapi_krb5.so.2,/usr/lib64/libldap_r-2.4.so.2,/usr/lib64/liblber-2.4.so.2,/usr/lib64/libunistring.so.2,/usr/lib64/libkrb5.so.3,/usr/lib64/libk5crypto.so.3,/lib64/libcom_err.so.2,/usr/lib64/libkrb5support.so.0,/lib64/libresolv.so.2,/usr/lib64/libsasl2.so.3,/usr/lib64/libkeyutils.so.1,/usr/lib64/libpcre.so.1" +-- authentication +singularity_bindpath = singularity_bindpath .. ",/usr/lib64/libmunge.so.2" +-- new additions for libfabric 1.15.2.0 +singularity_bindpath = singularity_bindpath .. ",/usr/lib64/libjitterentropy.so.3,/usr/lib64/libbrotlidec.so.1,/usr/lib64/libbrotlicommon.so.1,/usr/lib64/libjansson.so.4" +singularity_bindpath = singularity_bindpath .. ",/usr/lib64/libzstd.so.1" +singularity_bindpath = singularity_bindpath .. ",/lib64/libselinux.so.1" +-- lustre +singularity_bindpath = singularity_bindpath .. ",/usr/lib64/liblustreapi.so.1,/usr/lib64/liblnetconfig.so.4,/usr/lib64/libyaml-0.so.2,/usr/lib64/libnl-genl-3.so.200,/usr/lib64/libnl-3.so.200" -- this has to be conditional, path exists only in compute nodes if isDir("/var/spool/slurmd") then singularity_bindpath = singularity_bindpath .. ",/var/spool/slurmd" singularity_bindpath = singularity_bindpath .. ",/var/run/munge" end +-- add MPI END setenv("SINGULARITY_BINDPATH",singularity_bindpath) --- LD_PRELOAD edited to fix missing library dependency of Cray libmpi on libxpmem --- along with all libraries required for MPI to work on slingshot 10 --- This require updating once system is fully updated to slingshot 11 with compute nodes running cassini nics -prepend_path("SINGULARITYENV_LD_PRELOAD","/opt/cray/xpmem/default/lib64/libxpmem.so.0:/opt/cray/xpmem/default/lib64/libxpmem.so.0:/usr/lib64/librdmacm.so.1:/usr/lib64/libibverbs.so.1:/usr/lib64/libnl-3.so.200:/usr/lib64/libnl-route-3.so.200:/usr/lib64/libmlx5.so.1:/usr/lib64/libmunge.so.2") + +-- LD_PRELOAD addition +local singularity_ld_preload = "" +-- add MPI START +-- preload xpmem for fast mpi communication +singularity_ld_preload = singularity_ld_preload .. ":/opt/cray/xpmem/default/lib64/libxpmem.so.0" +-- singularity_ld_preload = singularity_ld_preload .. ":/lib64/libc.so.6:/lib64/libpthread.so.0:/lib64/librt.so.1:/lib64/libdl.so.2:/lib64/libz.so.1:/lib64/libselinux.so.1:/lib64/libm.so.6" +-- for Cassini nics and SS>=11 +singularity_ld_preload = singularity_ld_preload .. ":/usr/lib64/libcxi.so.1:/usr/lib64/libcurl.so.4:/usr/lib64/libjson-c.so.3" +singularity_ld_preload = singularity_ld_preload .. ":/usr/lib64/libnghttp2.so.14:/usr/lib64/libidn2.so.0:/usr/lib64/libssh.so.4:/usr/lib64/libpsl.so.5:/usr/lib64/libssl.so.1.1:/usr/lib64/libcrypto.so.1.1:/usr/lib64/libgssapi_krb5.so.2:/usr/lib64/libldap_r-2.4.so.2:/usr/lib64/liblber-2.4.so.2:/usr/lib64/libunistring.so.2:/usr/lib64/libkrb5.so.3:/usr/lib64/libk5crypto.so.3:/lib64/libcom_err.so.2:/usr/lib64/libkrb5support.so.0:/lib64/libresolv.so.2:/usr/lib64/libsasl2.so.3:/usr/lib64/libkeyutils.so.1:/usr/lib64/libpcre.so.1" +-- authentication +singularity_ld_preload = singularity_ld_preload .. ":/usr/lib64/libmunge.so.2" +-- new additions for libfabric 1.15.2.0 +singularity_ld_preload = singularity_ld_preload .. ":/usr/lib64/libjitterentropy.so.3:/usr/lib64/libbrotlidec.so.1:/usr/lib64/libbrotlicommon.so.1:/usr/lib64/libjansson.so.4" +singularity_ld_preload = singularity_ld_preload .. ":/usr/lib64/liblustreapi.so.1:/usr/lib64/liblnetconfig.so.4:/usr/lib64/libyaml-0.so.2:/usr/lib64/libnl-genl-3.so.200:/usr/lib64/libnl-3.so.200" +singularity_ld_preload = singularity_ld_preload .. ":/usr/lib64/libzstd.so.1" +singularity_ld_preload = singularity_ld_preload .. ":/lib64/libselinux.so.1" +-- lustre +singularity_ld_preload = singularity_ld_preload .. ":/usr/lib64/liblustreapi.so.1:/usr/lib64/liblnetconfig.so.4:/usr/lib64/libyaml-0.so.2:/usr/lib64/libnl-genl-3.so.200:/usr/lib64/libnl-3.so.200" +-- add MPI END +-- add GPUMPI START +singularity_ld_preload = singularity_ld_preload .. ":/opt/cray/pe/mpich/default/gtl/lib/libmpi_gtl_hsa.so.0" +-- add GPUMPI END +prepend_path("SINGULARITYENV_LD_PRELOAD", singularity_ld_preload) + +-- add GPUMPI START +setenv("MPICH_GPU_SUPPORT_ENABLED","1") +setenv("SINGULARITYENV_MPICH_GPU_SUPPORT_ENABLED","1") +-- add GPUMPI END + {% endif %} {% if spec.name == 'r' %}setenv("R_LIBS_USER", os.getenv("MYSOFTWARE").."/setonix/r/%v") {% endif %}