Move the containers build to CI

.github/workflows/containers.yaml

@@ -0,0 +1,120 @@
+name: Containers
+
+on:
+  push:
+    tags:
+      - "**"
+    branches:
+      - "**"
+    paths:
+      - "build/containers/**"
+      - "core"
+      - "combination"
+
+jobs:
+  checks:
+    name: Run quality checks
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Install requirements
+        run: |
+          sc_version="stable" # or "v0.4.7", or "latest"
+          hl_version="v2.12.0"
+          case $( uname -m ) in
+            arm64 | aarch64 )
+              sc_platform=aarch64
+              hl_platform=arm64
+              ;;
+            *)
+              sc_platform=$( uname -m )
+              hl_platform="${sc_platform}"
+              ;;
+          esac
+
+          echo "Installing shellcheck ..."
+          wget -qO- "https://github.com/koalaman/shellcheck/releases/download/${sc_version?}/shellcheck-${sc_version?}.linux.${sc_platform}.tar.xz" \
+            | tar -xJv >/dev/null
+          sudo cp "shellcheck-${sc_version}/shellcheck" /usr/bin/
+          shellcheck --version
+
+          echo "Installing hadolint ..."
+          wget -q https://github.com/hadolint/hadolint/releases/download/${hl_version}/hadolint-Linux-${hl_platform}
+          sudo cp "hadolint-Linux-${hl_platform}" /usr/bin/hadolint
+          sudo chmod 755 /usr/bin/hadolint
+          hadolint --version
+
+      - name: Run quality checks
+        run: |
+          make -C build/containers checks
+
+  list:
+    name: List containers
+    runs-on: ubuntu-latest
+    env:
+      IS_TAG: ${{ startsWith(github.ref, 'refs/tags/') }}
+    outputs:
+      containers: ${{ steps.list-containers.outputs.containers }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+
+      - id: list-containers
+        name: Determine containers
+        run: |
+          if [ "${IS_TAG}" == "false" ]; then
+            echo "Listing only the containers with changes"
+            containers=$( build/containers/build.sh --changed-containers-as-json)
+          elif [ "${IS_TAG}" == "true" ]; then
+            echo "Listing all the containers"
+            containers=$( build/containers/build.sh --all-containers-as-json)
+          fi
+          echo "containers = ${containers}"
+          echo "containers=${containers}" >> "$GITHUB_OUTPUT"
+
+  build:
+    name: Build
+    runs-on: ubuntu-latest
+    needs:
+      - checks
+      - list
+    strategy:
+      max-parallel: 8
+      fail-fast: false
+      matrix:
+        container: ${{ fromJSON(needs.list.outputs.containers) }}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Build Docker image
+        run: build/containers/build.sh --containers ${{ matrix.container }}
+
+  push:
+    name: Push
+    if: startsWith(github.ref, 'refs/tags/')
+    runs-on: ubuntu-latest
+    env:
+      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
+    needs:
+      - build
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Login to DockerHub
+        if: ${{ env.DOCKER_USERNAME != '' }}
+        run: echo ${{ secrets.DOCKER_PASSWORD }} | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin
+
+      - name: Push Docker images
+        if: ${{ env.DOCKER_USERNAME != '' }}
+        run: |
+          build/containers/build.sh --dry-run --skip-build --push --all-containers

build/Makefile

@@ -3,12 +3,10 @@ SHELL := /bin/bash
 .PHONY: all clean
 # trick to make all the first target
 # and set it to the end once all variables are defined
-all: containers datasets
+all: datasets
 
 build_dir = ..
 INTOGEN_DATASETS ?= ${build_dir}/datasets
-INTOGEN_CONTAINERS ?= ${build_dir}/containers
-
 
 # hg version
 genome = 38
@@ -17,17 +15,10 @@ cadd ?= 1.6
 # number of cores to use in steps that allow them
 cores ?= 1
 
-
 src_datasets = datasets
-src_containers = containers
-
-container_builder = ${src_containers}/build.sh
-
 
 # accumulate all targets
 DATASETS =
-CONTAINERS_SUDO =
-CONTAINERS_USER =
 
 # useful variables
 ensembl_db = "homo_sapiens_core_${ensembl}_${genome}"
@@ -39,9 +30,6 @@ grch = GRCh${genome}
 $(INTOGEN_DATASETS):
 	mkdir -p $@
 
-$(INTOGEN_CONTAINERS):
-	mkdir -p $@
-
 # Create checkpoints files so that if versions are changed
 # proper files are rebuild
 GENOME = $(INTOGEN_DATASETS)/.hg${genome}
@@ -55,23 +43,16 @@ $(ENSEMBL): | $(INTOGEN_DATASETS)
 $(CADD): | $(INTOGEN_DATASETS)
 	touch $@
 
-
-
 # Use second expansion for mixed dependencies
 .SECONDEXPANSION:
 
 include ${src_datasets}/*.mk
 include ${src_datasets}/*/*.mk
-include ${src_containers}/*/*.mk
 
 #########################
-.PHONY: datasets containers sudo
+.PHONY: datasets
 
 datasets: bgdata $(DATASETS) | $(INTOGEN_DATASETS)
 
-containers: $(CONTAINERS_USER) $(CONTAINERS_SUDO) | $(INTOGEN_CONTAINERS)
-
-sudo: $(CONTAINERS_SUDO) | $(INTOGEN_CONTAINERS)
-
 clean:
-	rm -rf $(INTOGEN_DATASETS) $(INTOGEN_CONTAINERS)
+	rm -rf $(INTOGEN_DATASETS)

build/containers/Makefile

@@ -0,0 +1,64 @@
+CURRENT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
+
+BUILD := $(CURRENT_DIR)/build.sh
+BUILD_ARGS ?=
+
+SHELLCHECK_BIN ?= shellcheck
+SHELLCHECK_ARGS ?= 
+
+HADOLINT_IGNORES := DL3059 # Multiple consecutive `RUN` instructions
+HADOLINT_IGNORES += DL3003 # Use WORKDIR to switch to a directory
+HADOLINT_IGNORES += DL3042 # Avoid use of cache directory with pip
+
+# HADOLINT_BIN := docker run --rm -i hadolint/hadolint hadolint
+HADOLINT_BIN ?= hadolint
+HADOLINT_ARGS ?= $(foreach ignore,$(HADOLINT_IGNORES),--ignore $(ignore))
+
+# Those ANSI codes are needed to print with colours
+GREEN := \033[0;32m
+WHITE := \033[0;37m
+BOLD := \033[1m
+BOLD_GREEN := $(GREEN)$(BOLD)
+BOLD_WHITE := $(WHITE)$(BOLD)
+RESET := \033[0m
+
+.DEFAULT_GOAL = checks
+
+.PHONY: checks
+checks: shellcheck hadolint
+
+.PHONY: shellcheck
+shellcheck:
+	@echo "$(BOLD_GREEN)Running cheks for shell scripts ...$(RESET)"
+	set -e; \
+	find -L $(CURRENT_DIR) -name '*.sh' | sed 's|$(CURRENT_DIR)/||' \
+		| while read -r script; do \
+		  echo "$(BOLD_WHITE)=> $${script}$(RESET)"; \
+		  $(SHELLCHECK_BIN) $(SHELLCHECK_ARGS) $${script}; \
+		  done
+
+.PHONY: hadolint
+hadolint:
+	@echo "$(BOLD_GREEN)Running checks for Dockerfile ...$(RESET)"
+	set -e; \
+	find -L $(CURRENT_DIR) -name 'Dockerfile' | sed 's|$(CURRENT_DIR)/||' \
+		| while read -r dockerfile; do \
+		  echo "$(BOLD_WHITE)=> $${dockerfile}$(RESET)"; \
+		  $(HADOLINT_BIN) $(HADOLINT_ARGS) - < $${dockerfile}; \
+		  done
+
+.PHONY: build-all
+build-all:
+	$(BUILD) --all-containers $(BUILD_ARGS)
+
+.PHONY: build-changes
+build-changes:
+	$(BUILD) $(BUILD_ARGS)
+
+.PHONY: push-all
+push-all:
+	$(BUILD) --push --all-containers $(BUILD_ARGS)
+
+.PHONY: push-changes
+push-changes:
+	$(BUILD) --push $(BUILD_ARGS)

build/containers/README.md

@@ -0,0 +1,61 @@
+# Containers
+
+This folder contains all the containers needed by the intogen pipelines specifically.
+
+Every container will have an independent folder here or a link to another folder in the repository. Every of those folders need to include a `Dockerfile` with the specs to build the Docker image.
+
+## CI/CD
+
+The Github workflow `containers.yaml` will run the following jobs:
+- Run some quality checks to make sure that we follow the best practices
+- Determine which containers need to be built and/or pushed:
+    - When the push corresponds with a tag, all the containers will be built and pushed.
+    - When this is a regular push into a branch, only the containers that were modified will be built to make sure that they are not broken by the changes.
+
+You can run the workflow locally with the following command:
+
+```shell
+act -W '.github/workflows/containers.yaml'
+```
+
+Note that you will need to install the tool [act](https://github.com/nektos/act).
+
+## Local Development
+
+You can run the following commands (note that you can skip the `-C build/containers` argument if you run the commands from the `build/containers` folder):
+
+### Running quality checks
+
+We run two types of checks:
+- [shellcheck](https://github.com/koalaman/shellcheck) to make sure that bash scripts follow the best practices
+- [hadolint](https://github.com/hadolint/hadolint) to make sure that Dockerfiles follow the best practices
+
+You will need to install them in your computer before running the following commands:
+
+- To run both checks:
+
+    ```shell
+    make -C build/containers checks
+    ```
+
+- Or individually with:
+
+    ```shell
+    make -C build/containers shellcheck
+    ```
+
+    ```shell
+    make -C build/containers hadolint
+    ```
+
+### Building all the containers
+
+```shell
+make -C build/containers build-all
+```
+
+### Building only the containers that were modified
+
+```shell
+make -C build/containers build-changed
+```