Build and push docker image to ghcr and run integration tests

fix and add test (#721) #895

Workflow file for this run

	name: Build and push docker image to ghcr and run integration tests

	on:
	workflow_dispatch:
	push:
	branches:
	- 'main'
	tags:
	- "v*"

	jobs:
	build-test-push-image:
	concurrency:
	group: ${{ github.workflow }}-${{ github.job }}-${{ github.head_ref \|\| github.run_id }}
	cancel-in-progress: true
	runs-on: a100-40gb-runner
	permissions:
	contents: write
	packages: write
	# This is used to complete the identity challenge
	# with sigstore/fulcio when running outside of PRs.
	id-token: write
	security-events: write

	steps:
	- name: Log in to GitHub Container Registry
	uses: docker/login-action@v1
	with:
	registry: ghcr.io
	username: ${{ github.repository_owner }}
	password: ${{ secrets.GHCR_PAT }}

	- name: Checkout repository
	uses: actions/checkout@v3
	with:
	submodules: recursive

	- name: Free Disk Space (Ubuntu)
	uses: jlumbroso/free-disk-space@main
	with:
	tool-cache: false
	android: true
	dotnet: true
	haskell: true
	large-packages: false
	swap-storage: true

	- name: Install soci
	uses: lerentis/[email protected]
	with:
	soci-release: "v0.4.0"

	- name: Set up Docker Buildx
	uses: docker/[email protected]

	- name: Set up containerd for ubuntu
	uses: crazy-max/[email protected]
	with:
	config-inline: \|
	version = 2

	# persistent data location
	root = "/runner/build/containerd"

	- name: Generate SHA tag
	id: meta
	uses: docker/metadata-action@v5
	with:
	images: \|
	ghcr.io/predibase/lorax
	tags: \|
	type=sha,prefix=,suffix=,format=short

	- name: Create a hash from tags
	env:
	tags: ${{ steps.meta.outputs.tags }}
	id: vars
	run: \|
	tag_hash=$(echo -n "$tags" \| md5sum \| awk '{print $1}')
	echo "tag_hash=$tag_hash" >> $GITHUB_OUTPUT
	echo "cache_dir=/runner/build/images/cache" >> $GITHUB_OUTPUT
	echo "image_dir=/runner/build/images" >> $GITHUB_OUTPUT
	echo "image_path=/runner/build/images/lorax" >> $GITHUB_OUTPUT

	- name: Create and update image/cache directory
	env:
	image_dir: ${{ steps.vars.outputs.image_dir }}
	cache_dir: ${{ steps.vars.outputs.cache_dir }}
	run: \|
	sudo mkdir -p $image_dir
	sudo chown ubuntu:ubuntu $image_dir

	sudo mkdir -p $cache_dir
	sudo chown ubuntu:ubuntu $cache_dir

	- name: Export Docker image as OCI
	uses: docker/build-push-action@v5
	with:
	context: .
	file: ./Dockerfile # Path to your Dockerfile
	push: false
	tags: ${{ steps.meta.outputs.tags }}
	load: true
	outputs: type=oci,compression=gzip,dest=${{ steps.vars.outputs.image_path }}-${{ steps.vars.outputs.tag_hash }}.tar.gz
	cache-from: type=local,src=${{ steps.vars.outputs.cache_dir }}
	cache-to: type=local,mode=max,image-manifest=true,oci-mediatypes=true,dest=${{ steps.vars.outputs.cache_dir }}

	# Uncomment below and comment the "Export Docker image as OCI" step
	# to test this workflow with an old image. This is useful for testing
	# the integration tests workflow without waiting for a new image to be built.

	# - name: Pull and retag an old image for testing with meta output tags
	# env:
	# tags: ${{ steps.meta.outputs.tags }}
	# run: \|
	# # Read the first tag from meta output
	# docker pull ghcr.io/predibase/lorax:main
	# # Tag with all the meta output tags
	# while IFS= read -r tag; do
	# echo "Tagging $tag"
	# docker tag ghcr.io/predibase/lorax:main "$tag"
	# done <<< "$tags"

	- name: Import image in containerd
	env:
	tag_hash: ${{ steps.vars.outputs.tag_hash }}
	image_path: ${{ steps.vars.outputs.image_path }}
	run: \|
	echo "Importing $image_path-$tag_hash to Containerd"
	sudo ctr i import --no-unpack --all-platforms --digests $image_path-$tag_hash.tar.gz

	- name: Push image with containerd
	env:
	tags: ${{ steps.meta.outputs.tags }}
	run: \|
	for tag in $tags
	do
	echo "Pushing $tag to GHCR"
	sudo ctr i push --user "${{ github.repository_owner }}:${{ secrets.GHCR_PAT }}" $tag
	done

	- name: Create and push soci index
	env:
	tags: ${{ steps.meta.outputs.tags }}
	run: \|
	export SOCI_PATH=$HOME/.soci/soci
	for tag in $tags
	do
	echo "Creating soci index for $tag"
	sudo $SOCI_PATH create $tag
	echo "Pushing soci index for $tag"
	sudo $SOCI_PATH push --user ${{ github.repository_owner }}:${{ secrets.GHCR_PAT }} $tag
	done

	- name: Run integration tests
	id: integration_tests
	uses: ./.github/workflows/integration-tests
	with:
	test_image_tag: ${{ steps.meta.outputs.tags }}
	use_local_image: true
	github_token: ${{ secrets.GHCR_PAT }}
	huggingface_token: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}

	- name: Final Tags
	id: final_meta
	uses: docker/metadata-action@v5
	with:
	images: \|
	ghcr.io/predibase/lorax
	tags: \|
	type=semver,pattern={{version}}
	type=semver,pattern={{major}}.{{minor}}
	type=sha,prefix=,suffix=,format=short
	type=raw,value=main,enable=${{ github.ref == 'refs/heads/main' }}

	- name: Tag the image with rest of the tags and push
	env:
	sha_tag: ${{ steps.meta.outputs.tags }}
	tags: ${{ steps.final_meta.outputs.tags }}
	run: \|
	# Read the first tag from meta output
	docker pull $sha_tag
	# Tag with all the meta output tags
	while IFS= read -r tag; do
	echo "Tagging $tag and pushing"
	docker tag $sha_tag "$tag"
	docker push "$tag"
	echo "Pushed $tag"
	done <<< "$tags"

	- name: Prune older images
	env:
	tag_hash: ${{ steps.vars.outputs.tag_hash }}
	image_path: ${{ steps.vars.outputs.image_path }}
	run: \|
	# Delete images older than a day from docker store
	docker image prune -a -f --filter "until=24h"

	# Delete the on disk copy
	rm -rf "$image_path-$tag_hash.tar.gz"

	# Delete the SHA image(s) from containerd store
	sudo ctr i rm $(sudo ctr i ls -q)

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

fix and add test (#721) #895

Workflow file

fix and add test (#721) #895

Jobs

Run details

Workflow file for this run