Skip to content

feat: simplify unify containers #115

feat: simplify unify containers

feat: simplify unify containers #115

Workflow file for this run

# Build all container images.
#
name: Build images
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
on:
pull_request:
paths-ignore:
- '.circleci/**'
- 'docs/**'
- 'test/**'
env:
# Used to override BIOCONDA_UTILS_VERSION in images/versions.sh
BIOCONDA_UTILS_VERSION: ${{ github.event.release && github.event.release.tag_name || github.head_ref || github.ref_name }}
jobs:
# JOBS FOR BUILDING IMAGES
# ----------------------------------------------------------------------
# These jobs will build images for archs, put them into a manifest, and push
# that to GitHub Container Registry. Later, the testing jobs will test and
# push to quay.io.
build-base-debian:
name: Build base-debian
runs-on: ubuntu-22.04
outputs:
# A note on these TAG_EXISTS_* outputs: these allow subsequent jobs to
# change behavior (e.g., skip building or skip pushing to ghcr) depending
# on whether an image has already been created.
TAG_EXISTS_base-debian: ${{ steps.base-debian.outputs.TAG_EXISTS_base-debian }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Install qemu dependency
run: |
sudo apt-get update
sudo apt-get install -y qemu-user-static
- name: base-debian
id: base-debian
run: |
source images/versions.sh
if [ $(tag_exists $BASE_DEBIAN_IMAGE_NAME $BASE_TAG) ]; then
echo "TAG_EXISTS_base-debian=true" >> $GITHUB_OUTPUT
else
cd images && bash build.sh base-glibc-debian-bash
fi
- name: push to ghcr
if: '${{ ! steps.base-debian.outputs.TAG_EXISTS_base-debian }}'
run: |
echo '${{ secrets.GITHUB_TOKEN }}' | podman login ghcr.io -u '${{ github.actor }}' --password-stdin
source images/versions.sh
push_to_ghcr $BASE_DEBIAN_IMAGE_NAME $BASE_TAG
build-base-busybox:
name: Build base-busybox
runs-on: ubuntu-22.04
outputs:
TAG_EXISTS_base-busybox: ${{ steps.base-busybox.outputs.TAG_EXISTS_base-busybox }}
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Install qemu dependency
run: |
sudo apt-get update
sudo apt-get install -y qemu-user-static
- name: base-busybox
id: base-busybox
run: |
source images/versions.sh
if [ $(tag_exists $BASE_BUSYBOX_IMAGE_NAME $BASE_TAG) ]; then
echo "TAG_EXISTS_base-busybox=true" >> $GITHUB_OUTPUT
else
cd images && bash build.sh base-glibc-busybox-bash
fi
- name: push to ghcr
if: '${{ ! steps.base-busybox.outputs.TAG_EXISTS_base-busybox }}'
run: |
echo '${{ secrets.GITHUB_TOKEN }}' | podman login ghcr.io -u '${{ github.actor }}' --password-stdin
source images/versions.sh
push_to_ghcr $BASE_BUSYBOX_IMAGE_NAME $BASE_TAG
build-build-env:
name: Build build-env
outputs:
TAG_EXISTS_build-env: ${{ steps.build-env.outputs.TAG_EXISTS_build-env }}
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Install qemu dependency
run: |
sudo apt-get update
sudo apt-get install -y qemu-user-static
- name: build-env
id: build-env
run: |
source images/versions.sh
if [ $(tag_exists $BUILD_ENV_IMAGE_NAME $BIOCONDA_IMAGE_TAG) ]; then
echo "TAG_EXISTS_build-env=true" >> $GITHUB_OUTPUT
else
cd images && bash build.sh bioconda-utils-build-env-cos7
fi
- name: push to ghcr
if: '${{ ! steps.build-env.outputs.TAG_EXISTS_build-env }}'
run: |
echo '${{ secrets.GITHUB_TOKEN }}' | podman login ghcr.io -u '${{ github.actor }}' --password-stdin
source images/versions.sh
push_to_ghcr $BUILD_ENV_IMAGE_NAME $BIOCONDA_IMAGE_TAG
build-create-env:
name: Build create-env
needs: [build-build-env, build-base-busybox]
outputs:
TAG_EXISTS_create-env: ${{ steps.create-env.outputs.TAG_EXISTS_create-env }}
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Install qemu dependency
run: |
sudo apt-get update
sudo apt-get install -y qemu-user-static
- name: Build create-env
id: create-env
run: |
source images/versions.sh
echo '${{ secrets.GITHUB_TOKEN }}' | podman login ghcr.io -u '${{ github.actor }}' --password-stdin
if [ $(tag_exists $CREATE_ENV_IMAGE_NAME $BIOCONDA_IMAGE_TAG) ]; then
echo "TAG_EXISTS_create-env=true" >> $GITHUB_OUTPUT
else
cd images && bash build.sh create-env
fi
- name: push to ghcr
if: '${{ ! steps.create-env.outputs.TAG_EXISTS_create-env }}'
run: |
echo '${{ secrets.GITHUB_TOKEN }}' | podman login ghcr.io -u '${{ github.actor }}' --password-stdin
source images/versions.sh
push_to_ghcr $CREATE_ENV_IMAGE_NAME $BIOCONDA_IMAGE_TAG
# END OF BUILDING IMAGES
# ----------------------------------------------------------------------
# START TESTING
# These testing jobs will run the respective Dockerfile.test in each image
# directory.
test:
name: test bioconda-utils with images
runs-on: ubuntu-20.04
needs: [build-base-debian, build-base-busybox, build-build-env, build-create-env]
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
# # Clone bioconda-recipes to use as part of the tests.
# - uses: actions/checkout@v4
# with:
# fetch-depth: 0
# repository: bioconda/bioconda-recipes
# path: recipes
# - name: set path
# run: echo "/opt/mambaforge/bin" >> $GITHUB_PATH
- name: Install bioconda-utils
run: |
export BIOCONDA_DISABLE_BUILD_PREP=1
BRANCH=simplify-unify-containers
wget https://raw.githubusercontent.com/bioconda/bioconda-common/${BRANCH}/{common,install-and-set-up-conda,configure-conda}.sh
source images/versions.sh
# Ensure install-and-set-up-conda uses same version as in the container
# (which uses images/versions.sh
export BIOCONDA_UTILS_TAG=$BIOCONDA_UTILS_VERSION
bash install-and-set-up-conda.sh
eval "$(conda shell.bash hook)"
conda create -n bioconda -y --file test-requirements.txt --file bioconda_utils/bioconda_utils-requirements.txt
conda activate bioconda
python setup.py install
- name: test
run: |
eval "$(conda shell.bash hook)"
conda activate bioconda
source images/versions.sh
# Figure out which registry to use for each image, based on what was built.
[ ${{ needs.build-build-env.outputs.TAG_EXISTS_build-env }} ] && BUILD_ENV_REGISTRY='quay.io/bioconda' || BUILD_ENV_REGISTRY="ghcr.io/bioconda"
[ ${{ needs.build-create-env.outputs.TAG_EXISTS_create-env }} ] && CREATE_ENV_REGISTRY='quay.io/bioconda' || CREATE_ENV_REGISTRY="ghcr.io/bioconda"
[ ${{ needs.build-base-busybox.outputs.TAG_EXISTS_base_busybox }} ] && DEST_BASE_REGISTRY='quay.io/bioconda' || DEST_BASE_REGISTRY="ghcr.io/bioconda"
[ ${{ needs.build-base-debian.outputs.TAG_EXISTS_base_debian }} ] && DEST_EXTENDED_BASE_REGISTRY='quay.io/bioconda' || DEST_EXTENDED_BASE_REGISTRY="ghcr.io/bioconda"
# Tell mulled-build which image to use
export DEST_BASE_IMAGE="${DEST_BASE_IMAGE_REGISTRY}/${BASE_BUSYBOX_IMAGE_NAME}:${BASE_TAG}"
export DEFAULT_BASE_IMAGE="${DEST_BASE_REGISTRY}/${BASE_BUSYBOX_IMAGE_NAME}"
export DEFAULT_EXTENDED_BASE_IMAGE="${DEST_EXTENDED_BASE_REGISTRY}/${BASE_DEBIAN_IMAGE_NAME}"
export BUILD_ENV_IMAGE="${BUILD_ENV_REGISTRY}/${BUILD_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}"
export CREATE_ENV_IMAGE="${CREATE_ENV_REGISTRY}/${CREATE_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}"
# # Build a package with containers.
# cd recipes
# bioconda-utils build \
# --docker-base-image "${BUILD_ENV_REGISTRY}/${BUILD_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}" \
# --mulled-conda-image "${CREATE_ENV_REGISTRY}/${CREATE_ENV_IMAGE_NAME}:${BIOCONDA_IMAGE_TAG}" \
# --packages seqtk \
# --docker \
# --mulled-test \
# --force
py.test --durations=0 test/ -v --log-level=DEBUG -k "docker" --tb=native
# END TESTING
# ------------------------------------------------------------------------
# START PUSHING IMAGES
# For these push steps, a repository must first exist on quay.io/bioconda
# AND that repository must also be configured to allow write access for the
# appropriate service account. This must be done by a user with admin
# access to quay.io/bioconda.
#
# This uses the TAG_EXISTS_* outputs from previous jobs to determine if
# a push to quay.io should happen.
push:
name: push images
runs-on: ubuntu-20.04
needs: [build-base-debian, build-base-busybox, build-build-env, build-create-env, test]
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
- name: push base-debian
if: ${{ ! needs.base-debian.outputs.TAG_EXISTS_base-debian }}
run: |
echo '${{ secrets.QUAY_BIOCONDA_TOKEN }}' | podman login quay.io -u '${{ secrets.QUAY_BIOCONDA_USERNAME }}' --password-stdin
source images/versions.sh
move_from_ghcr_to_quay ${BASE_DEBIAN_IMAGE_NAME} ${BASE_TAG}
- name: push base-busybox
if: ${{ ! needs.base-busybox.outputs.TAG_EXISTS_base-busybox }}
run: |
echo '${{ secrets.QUAY_BIOCONDA_TOKEN }}' | podman login quay.io -u '${{ secrets.QUAY_BIOCONDA_USERNAME }}' --password-stdin
source images/versions.sh
move_from_ghcr_to_quay ${BASE_BUSYBOX_IMAGE_NAME} ${BASE_TAG}
- name: push create-env
if: ${{ ! needs.create-env.outputs.TAG_EXISTS_create-env }}
run: |
echo '${{ secrets.QUAY_BIOCONDA_TOKEN }}' | podman login quay.io -u '${{ secrets.QUAY_BIOCONDA_USERNAME }}' --password-stdin
source images/versions.sh
move_from_ghcr_to_quay ${CREATE_ENV_IMAGE_NAME} ${BIOCONDA_IMAGE_TAG}
- name: push build-env
if: ${{ ! needs.build-env.outputs.TAG_EXISTS_build-env }}
run: |
echo '${{ secrets.QUAY_BIOCONDA_TOKEN }}' | podman login quay.io -u '${{ secrets.QUAY_BIOCONDA_USERNAME }}' --password-stdin
source images/versions.sh
move_from_ghcr_to_quay ${BUILD_ENV_IMAGE_NAME} ${BIOCONDA_IMAGE_TAG}