From 74bd026e45d6e0971a3096bca7b7b9cbc1f1d96e Mon Sep 17 00:00:00 2001 From: Hilmar Lapp Date: Tue, 8 Oct 2024 20:19:47 -0400 Subject: [PATCH] Updates and renames containerization lesson from numbered to named --- Lesson-02.qmd => Lesson-Contain.qmd | 75 +++++----- pub/{Lesson-02.html => Lesson-Contain.html} | 136 +++++++++--------- .../libs/clipboard/clipboard.min.js | 0 .../libs/quarto-html/light-border.css | 0 .../libs/quarto-html/popper.min.js | 0 .../libs/quarto-html/quarto-html.min.css | 0 .../quarto-syntax-highlighting.css | 0 .../libs/quarto-html/tabby.min.js | 0 .../libs/quarto-html/tippy.css | 0 .../libs/quarto-html/tippy.umd.min.js | 0 .../libs/revealjs/dist/reset.css | 0 .../libs/revealjs/dist/reveal.css | 0 .../libs/revealjs/dist/reveal.esm.js | 0 .../libs/revealjs/dist/reveal.esm.js.map | 0 .../libs/revealjs/dist/reveal.js | 0 .../libs/revealjs/dist/reveal.js.map | 0 .../dist/theme/fonts/league-gothic/LICENSE | 0 .../fonts/league-gothic/league-gothic.css | 0 .../fonts/league-gothic/league-gothic.eot | Bin .../fonts/league-gothic/league-gothic.ttf | Bin .../fonts/league-gothic/league-gothic.woff | Bin .../dist/theme/fonts/source-sans-pro/LICENSE | 0 .../source-sans-pro-italic.eot | Bin .../source-sans-pro-italic.ttf | Bin .../source-sans-pro-italic.woff | Bin .../source-sans-pro-regular.eot | Bin .../source-sans-pro-regular.ttf | Bin .../source-sans-pro-regular.woff | Bin .../source-sans-pro-semibold.eot | Bin .../source-sans-pro-semibold.ttf | Bin .../source-sans-pro-semibold.woff | Bin .../source-sans-pro-semibolditalic.eot | Bin .../source-sans-pro-semibolditalic.ttf | Bin .../source-sans-pro-semibolditalic.woff | Bin .../fonts/source-sans-pro/source-sans-pro.css | 0 .../libs/revealjs/dist/theme/quarto.css | 0 .../plugin/highlight/highlight.esm.js | 0 .../revealjs/plugin/highlight/highlight.js | 0 .../revealjs/plugin/highlight/monokai.css | 0 .../libs/revealjs/plugin/highlight/plugin.js | 0 .../revealjs/plugin/highlight/zenburn.css | 0 .../revealjs/plugin/markdown/markdown.esm.js | 0 .../libs/revealjs/plugin/markdown/markdown.js | 0 .../libs/revealjs/plugin/markdown/plugin.js | 0 .../libs/revealjs/plugin/math/katex.js | 0 .../libs/revealjs/plugin/math/math.esm.js | 0 .../libs/revealjs/plugin/math/math.js | 0 .../libs/revealjs/plugin/math/mathjax2.js | 0 .../libs/revealjs/plugin/math/mathjax3.js | 0 .../libs/revealjs/plugin/math/plugin.js | 0 .../libs/revealjs/plugin/notes/notes.esm.js | 0 .../libs/revealjs/plugin/notes/notes.js | 0 .../libs/revealjs/plugin/notes/plugin.js | 0 .../revealjs/plugin/notes/speaker-view.html | 0 .../revealjs/plugin/pdf-export/pdfexport.js | 0 .../revealjs/plugin/pdf-export/plugin.yml | 0 .../quarto-line-highlight/line-highlight.css | 0 .../quarto-line-highlight/line-highlight.js | 0 .../plugin/quarto-line-highlight/plugin.yml | 0 .../revealjs/plugin/quarto-support/footer.css | 0 .../revealjs/plugin/quarto-support/plugin.yml | 0 .../revealjs/plugin/quarto-support/support.js | 0 .../libs/revealjs/plugin/reveal-menu/menu.css | 0 .../libs/revealjs/plugin/reveal-menu/menu.js | 0 .../revealjs/plugin/reveal-menu/plugin.yml | 0 .../plugin/reveal-menu/quarto-menu.css | 0 .../plugin/reveal-menu/quarto-menu.js | 0 .../libs/revealjs/plugin/search/plugin.js | 0 .../libs/revealjs/plugin/search/search.esm.js | 0 .../libs/revealjs/plugin/search/search.js | 0 .../libs/revealjs/plugin/zoom/plugin.js | 0 .../libs/revealjs/plugin/zoom/zoom.esm.js | 0 .../libs/revealjs/plugin/zoom/zoom.js | 0 73 files changed, 103 insertions(+), 108 deletions(-) rename Lesson-02.qmd => Lesson-Contain.qmd (74%) rename pub/{Lesson-02.html => Lesson-Contain.html} (87%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/clipboard/clipboard.min.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/quarto-html/light-border.css (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/quarto-html/popper.min.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/quarto-html/quarto-html.min.css (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/quarto-html/quarto-syntax-highlighting.css (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/quarto-html/tabby.min.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/quarto-html/tippy.css (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/quarto-html/tippy.umd.min.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/reset.css (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/reveal.css (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/reveal.esm.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/reveal.esm.js.map (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/reveal.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/reveal.js.map (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/theme/fonts/league-gothic/LICENSE (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/theme/fonts/league-gothic/league-gothic.css (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/theme/fonts/league-gothic/league-gothic.eot (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/theme/fonts/league-gothic/league-gothic.ttf (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/theme/fonts/league-gothic/league-gothic.woff (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/theme/fonts/source-sans-pro/LICENSE (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/theme/fonts/source-sans-pro/source-sans-pro-italic.eot (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/theme/fonts/source-sans-pro/source-sans-pro-italic.ttf (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/theme/fonts/source-sans-pro/source-sans-pro-italic.woff (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/theme/fonts/source-sans-pro/source-sans-pro-regular.eot (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/theme/fonts/source-sans-pro/source-sans-pro-regular.ttf (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/theme/fonts/source-sans-pro/source-sans-pro-regular.woff (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/theme/fonts/source-sans-pro/source-sans-pro-semibold.eot (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/theme/fonts/source-sans-pro/source-sans-pro-semibold.ttf (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/theme/fonts/source-sans-pro/source-sans-pro-semibold.woff (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/theme/fonts/source-sans-pro/source-sans-pro-semibolditalic.eot (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/theme/fonts/source-sans-pro/source-sans-pro-semibolditalic.ttf (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/theme/fonts/source-sans-pro/source-sans-pro-semibolditalic.woff (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/theme/fonts/source-sans-pro/source-sans-pro.css (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/dist/theme/quarto.css (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/highlight/highlight.esm.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/highlight/highlight.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/highlight/monokai.css (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/highlight/plugin.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/highlight/zenburn.css (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/markdown/markdown.esm.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/markdown/markdown.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/markdown/plugin.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/math/katex.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/math/math.esm.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/math/math.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/math/mathjax2.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/math/mathjax3.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/math/plugin.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/notes/notes.esm.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/notes/notes.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/notes/plugin.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/notes/speaker-view.html (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/pdf-export/pdfexport.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/pdf-export/plugin.yml (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/quarto-line-highlight/line-highlight.css (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/quarto-line-highlight/line-highlight.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/quarto-line-highlight/plugin.yml (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/quarto-support/footer.css (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/quarto-support/plugin.yml (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/quarto-support/support.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/reveal-menu/menu.css (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/reveal-menu/menu.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/reveal-menu/plugin.yml (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/reveal-menu/quarto-menu.css (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/reveal-menu/quarto-menu.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/search/plugin.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/search/search.esm.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/search/search.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/zoom/plugin.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/zoom/zoom.esm.js (100%) rename pub/{Lesson-02_files => Lesson-Contain_files}/libs/revealjs/plugin/zoom/zoom.js (100%) diff --git a/Lesson-02.qmd b/Lesson-Contain.qmd similarity index 74% rename from Lesson-02.qmd rename to Lesson-Contain.qmd index 46c9b40..31e6ae0 100644 --- a/Lesson-02.qmd +++ b/Lesson-Contain.qmd @@ -2,7 +2,7 @@ title: "Biostat 823 - Containerization" author: "Hilmar Lapp" institute: "Duke University, Department of Biostatistics & Bioinformatics" -date: "Sep 14, 2023" +date: "Oct 10, 2024" format: revealjs: slide-number: true @@ -23,7 +23,7 @@ Reproducibility of computational research faces four major challenges^[Boettiger ## "Dependency Hell" -* Software dependencies have themselves dependencies recursively +* Software dependencies have themselves dependencies, recursively * Dependencies can be be often difficult to install (require compilation, manual "tweaks" due local OS or other differences, etc) * Required version may conflict with that required by other software, or may not work with the local OS version, making it impossible to install. - The likelihood of conflicts is particularly high on shared computing environments. @@ -42,7 +42,7 @@ Reproducibility of computational research faces four major challenges^[Boettiger - This can happen anywhere in the dependency chain. * Dependencies can also become unmaintained or end-of-life - Can result in removal from package repositories. - - Python 2.x example + - Python 2.x example; CRAN package removal by policy ## Virtual Machine as solution? @@ -65,10 +65,10 @@ Reproducibility of computational research faces four major challenges^[Boettiger - 2007 (v1) and 2013--16 (v2): Linux control groups ([cgroups](https://en.wikipedia.org/wiki/Cgroups)) - 2008: [Linux Containers (LXC)](https://en.wikipedia.org/wiki/LXC) - 2013: [Docker](https://www.docker.com) -- 2015: [Singularity](https://en.wikipedia.org/wiki/Singularity_(software)) +- 2015: [Singularity](https://en.wikipedia.org/wiki/Singularity_(software)) and ([since 2021](https://apptainer.org/news/community-announcement-20211130/)) [Apptainer](https://apptainer.org) ::: aside -There are many [OS-level virtualization](https://en.wikipedia.org/wiki/OS-level_virtualization) systems. LXC, Docker, and Singularity are by far the most important ones. +There are many [OS-level virtualization](https://en.wikipedia.org/wiki/OS-level_virtualization) systems. LXC, and especially *Docker*, and *Apptainer/Singularity* are by far the most important ones. ::: ## Properties of containerized processes {.smaller} @@ -83,8 +83,8 @@ There are many [OS-level virtualization](https://en.wikipedia.org/wiki/OS-level_ ![](images/docker-for-mac.png){fig-align="right" width="30%" style="float: right"}

* On Windows and macOS, requires a Linux VM - - Part of the Docker installation (uses WSL on Windows; LinuxKit / Hypervisor Framework on macOS) - - Unsupported by Singularity + - Part of the Docker installation (uses [WSL/WSL2](https://learn.microsoft.com/en-us/windows/wsl/about) on Windows; LinuxKit / Hypervisor Framework on macOS) + - Apptainer can [use WSL/WSL2 on Windows](https://apptainer.org/docs/admin/main/installation.html#windows), with access to GPUs; [on macOS](https://apptainer.org/docs/admin/main/installation.html#mac), requires [Lima](https://lima-vm.io) as VM host (no GPU) ::: aside Figure modified from [Gianluca Quercini, Cloud computing -- Docker Primer](https://gquercini.github.io/courses/cloud-computing/references/docker-primer/) @@ -98,18 +98,19 @@ Figure modified from [Gianluca Quercini, Cloud computing -- Docker Primer](https From [ELIXIR containers nextflow: Docker](https://biocorecrg.github.io/ELIXIR_containers_nextflow/docker.html) ::: -## Singularity: Containers for HPC +## Apptainer / Singularity: Containers for HPC {.smaller} * HPC systems are shared computing environments - Docker daemon runs as root, processes within container can run as root - Not permissible on a shared computing environment -* Singularity does not require elevated privileges +* Apptainer does not require elevated privileges - Launcher run by user, not a daemon run by root - Processes inside container run as same user as outside -* Singularity containers can be built (bootstrapped) from (many) Docker container images +* Apptainer containers can be built (bootstrapped) from (many) Docker container images - Most scientific software containers are compatible + - Issues can occur for containers that run services under a privileged user (httpd, database server, etc) -## Singularity architecture vs Docker +## Apptainer / Singularity vs Docker ![](images/singularity_architecture.png) @@ -162,15 +163,6 @@ CMD ["java", "-jar", "picard.jar"] - [GitHub Packages](https://ghcr.io) Repository (includes container images) - Gitlab container registry (gitlab-registry.oit.duke.edu for Duke OIT's Gitlab installation) -## (Note) Container images are layered - -* Container file system is a [union mount](https://en.wikipedia.org/wiki/Union_mount) - - [OverlayFS](https://en.wikipedia.org/wiki/OverlayFS) supported by Linux kernel since 2014 - - Allows layering image content - - Each command in the definition creates a layer - - Layers are cached for image builds and pulls -* [Best practices for container definition](https://docs.docker.com/develop/develop-images/dockerfile_best-practices/) include controlling layer cache invalidation - ## (Note) Multi-stage builds * Build layers are read-only @@ -179,39 +171,50 @@ CMD ["java", "-jar", "picard.jar"] - Multiple container builds in one container definition - Use to retain build products but not the software environment needed to create them (which can be large) -## (Note) Build docker, run singularity +## (Note) Build docker, run apptainer -* Building Docker images typically more flexible - - No Singularity Desktop version for Windows or macOS (requires Linux VM instead) - - `singularity build` normally requires `sudo` privileges -* Singularity can use (most) Docker images directly +* Building Docker container images typically more flexible + - No Apptainer Desktop version for Windows or macOS (requires Linux VM instead) + - Container build instructions may cause problems with `apptainer build` in unprivileged environment (which uses `--fakeroot` by default) +* Apptainer can use (most) Docker images directly - Can download and run in one step: ```shell - $ singularity run docker:// + $ apptainer run docker:// ``` -* Use `--fakeroot` for `singularity build` in a non-privileged environment -## (Note) Mounting data into the container +## (Note) Mounting data into container {.smaller} Requires bind mount at container runtime (`docker run`): -* `--volume :` (Docker) -* `--bind :` (Singularity) +* [Docker](https://docs.docker.com/engine/storage/bind-mounts/): + + `--volume :` + + or + + `--mount type=bind,source=,target=` + + Using `--mount` generates an error if target directory (or file) doesn't exist + +* [Apptainer](https://apptainer.org/docs/user/main/bind_paths_and_mounts.html#user-defined-bind-paths): + + `--bind :` + + Or use `--mount` (see above). * Can be used for directories and files -* Using `--mount` generates an error if target directory (or file) doesn't exist ## Resources (I) * [Dockerfile reference](https://docs.docker.com/engine/reference/builder/) * [Docker command line reference](https://docs.docker.com/engine/reference/commandline/cli/) -* [Singularity file reference](https://docs.sylabs.io/guides/latest/user-guide/definition_files.html) -* [Singularity command line reference](https://docs.sylabs.io/guides/latest/user-guide/cli.html) -* [Open Containers Initiative (OCI) standard for annotations](https://github.com/opencontainers/image-spec/blob/main/annotations.md) +* [Apptainer file reference](https://apptainer.org/docs/user/main/definition_files.html) +* [Apptainer command line reference](https://apptainer.org/docs/user/main/cli.html) +* [Open Containers Initiative (OCI) standard for annotations](https://specs.opencontainers.org/image-spec/annotations/) ## Resources (II) -* [Introduction to Docker](https://carpentries-incubator.github.io/docker-introduction/) (Carpentries Incubator lesson) +* [Intro to Docker Workshop](https://imageomics.github.io/docker-workshop/) (Based on Carpentries Incubator lesson) +* [Into to Singularity Workshop](https://carpentries-incubator.github.io/singularity-introduction/) (Carpentries Incubator lesson) * [DCC OnDemand](https://dcc-ondemand-01.oit.duke.edu/) * [Jupyter Docker Stacks](https://jupyter-docker-stacks.readthedocs.io/) - Customized [Biostat Jupyter Docker container](https://github.com/Duke-GCB/biostat-jupyter) -* [Biostat-823 "everything" GPU container](https://gitlab.oit.duke.edu/owzar001/bios-823-container-gpu/-/blob/main/README.md) (Singularity) \ No newline at end of file diff --git a/pub/Lesson-02.html b/pub/Lesson-Contain.html similarity index 87% rename from pub/Lesson-02.html rename to pub/Lesson-Contain.html index 5fdfd1f..9df4aa3 100644 --- a/pub/Lesson-02.html +++ b/pub/Lesson-Contain.html @@ -1,23 +1,23 @@ - - - - - - - - - + + + + + + + + + - + Biostat 823 - Containerization - - + + - - - - - + + + + +