CI #284

Workflow file for this run

	# name of the workflow. Link to the documentation - https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#name
	name: CI
	# running on push to main and develop branches or on pull reuqests or on manual trigger
	on:
	# manual trigger
	workflow_dispatch:
	inputs:
	ssh_debug_enabled:
	type: boolean
	description: 'Run the build/test with ssh debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
	required: false
	default: false
	debug_deployment:
	type: boolean
	description: 'Run the pipeline with debug deployment enabled'
	required: false
	default: false

	# runnnig on push to main and develop branches
	push:
	branches:
	- main
	- develop
	paths-ignore:
	- '**/README.md'
	- '.devcontainer/**'
	# running on pull requests to main and develop branches
	pull_request:
	branches:
	- main
	- develop
	paths-ignore:
	- '**/README.md'

	# defining global environment variables for all jobs
	env:
	# define runner indexes for tests splitting and parallel execution
	total-runners: 5
	# defining GitHub registry for docker images
	REGISTRY: ghcr.io
	# github.repository as <account>/<repo>
	IMAGE_NAME: ${{ github.repository }}

	# build job definition with 7 steps:
	# 1. Checkout repository
	# 2. Cache Maven packages
	# 3. Running Liquibase Quality Checks link - https://docs.liquibase.com/tools-integrations/liquibase-quality-checks/overview.html
	# 4. Initialize CodeQL
	# 5. Build with Maven
	# 6. Perform CodeQL Analysis link - https://docs.github.com/en/code-security/secure-coding/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-code-scanning
	# 7. Upload artifact
	jobs:
	build:
	# build job will run on ubuntu-latest github-hosted runner
	runs-on: ubuntu-latest
	# defining permissions for the job - read contents, write packages, write id-token. Link to the documentation - https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idpermissions
	# security-events permission is required for CodeQL analysis
	# enforcing policy for the job - only users with write access to the repository can trigger the job. Link to the documentation - https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idenforce_admins
	permissions:
	actions: read
	contents: read
	packages: write
	id-token: write
	security-events: write
	# defining strategy for the job - matrix strategy for codeQL analysis
	strategy:
	fail-fast: false
	matrix:
	language: [ 'java' ]
	# CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
	# Use only 'java' to analyze code written in Java, Kotlin or both
	# Use only 'javascript' to analyze code written in JavaScript, TypeScript or both
	# Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support

	# defining steps for the job as explained above
	steps:
	- name: Checkout repository
	uses: actions/checkout@v3 # cache maven packages step - caching maven packages to speed up the build process. Link to the documentation - https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows
	- name: Cache Maven packages
	uses: actions/cache@v3 # defining cache key and restore keys for the cache step. Link to the documentation - https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows#matching-a-cache-key
	with:
	path: ~/.m2/repository # path to the directory where maven packages are stored - /root/.m2 in the container
	key: ${{ runner.os }}-build-${{ hashFiles('**/pom.xml') }}
	restore-keys: ${{ runner.os }}-build-
	# running liquibase quality checks step to ensure that the database changelogs are well-formed and follow best practices.
	# Link to the documentation - https://docs.liquibase.com/tools-integrations/liquibase-quality-checks/overview.html
	# Using the Liquibase Maven plugin, you can run Liquibase Quality Checks on your changelogs.
	# The plugin is available in the Maven Central repository. Link to Maven Central - https://search.maven.org/artifact/org.liquibase/liquibase-maven-plugin
	- name: Running Liquibase Quality Checks to ensure that the database changelogs are well-formed and follow best practices
	run: \|
	mvn process-resources liquibase:checks.run
	# runnning code scanning with CodeQL. Link to the documentation - https://docs.github.com/en/code-security/secure-coding/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-code-scanning
	# first step is to initialize CodeQL
	- name: Initialize CodeQL
	uses: github/codeql-action/init@v3
	with:
	languages: ${{ matrix.language }} # defining the language for the CodeQL analysis
	# debug: true # uncomment this line to enable debugging for CodeQL analysis step
	# If you wish to specify custom queries, you can do so here or in a config file.
	# By default, queries listed here will override any specified in a config file.
	# Prefix the list here with "+" to use these queries and those in the config file.

	# Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
	# queries: security-extended,security-and-quality

	# autobuild with codeql
	# Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java).
	# If this step fails, then you should remove it and run the build manually (see below)
	- name: Autobuild
	uses: github/codeql-action/autobuild@v3

	# performing Code Quality Analysis with CodeQL. Link to the documentation - https://docs.github.com/en/code-security/secure-coding/automatically-scanning-your-code-for-vulnerabilities-and-errors/about-code-scanning
	- name: Perform CodeQL Analysis
	uses: github/codeql-action/analyze@v3
	with:
	category: "/language:${{matrix.language}}" # defining the language for the CodeQL analysis
	- uses: actions/upload-artifact@v3 # uploading the artifact to the GitHub Artifacts. Link to the documentation - https://docs.github.com/en/actions/guides/storing-workflow-data-as-artifacts
	with:
	name: jar-artifact # naming the artifact jar file/s path
	path: target/ # actual relative path to the artifact in the container - target/

	unit-parallel-tests:
	# unit-parallel-tests job will run on ubuntu-latest github-hosted runner
	name: UNIT-PARALLEL-TESTS
	runs-on: ubuntu-latest
	needs: # needs build job and runner-indexes job to be completed before running the unit-parallel-tests job
	- build
	- runner-indexes
	container:
	image: ghcr.io/tsviz/tsvi-spring-test:v1.0.0 # using the base maven:3.8.7-openjdk-18 image + jq + curl + nodejs 18
	services:
	# postgres service container
	postgres: # service name - postgres. This name is used to access the service container from the job container as the host name.
	image: postgres # running the job in a container - postgres link to the docker hub - https://hub.docker.com/_/postgres
	env:
	POSTGRES_PASSWORD: postgres # setting the password for the postgres database
	# exposing the port 5432 of the postgres service container to the host machine
	ports:
	- 5432:5432
	# redis service container for caching session data
	redis: # service name - redis. This name is used to access the service container from the job container as the host name.
	image: redis # running the job in a container - redis link to the docker hub - https://hub.docker.com/_/redis
	# exposing the port 6379 of the redis service container to the host machine
	ports:
	- 6379:6379
	# defining the job permissions
	permissions:
	contents: read # read access to the repository contents
	packages: write # write access to the repository packages
	id-token: write # write access to the repository id token
	strategy: # defining the job to run in parallel with the matrix strategy and runner-indexes job output
	fail-fast: true # cancels all in-progress jobs if any matrix job fails link to the documentation - https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjob_idstrategyfail-fast
	matrix: # defining the matrix strategy to run the job in parallel using x number of github-hosted runners defined in the env total_runners above
	runner-index: ${{ fromjson(needs.runner-indexes.outputs.json) }} # using the runner-indexes job output to define the matrix strategy
	steps:
	- name: Checkout repository # checkout the repository
	uses: actions/[email protected]
	# caching the maven packages to speed up the build process.
	# Link to the documentation - https://docs.github.com/en/actions/guides/caching-dependencies-to-speed-up-workflows
	- name: Cache Maven packages
	uses: actions/cache@v3 # using the actions/cache@v3 action to cache the maven packages
	with:
	path: /root/.m2 # path to cache
	key: ${{ runner.os }}-junit-${{ hashFiles('**/pom.xml') }} # key for restoring and saving the cache
	restore-keys: ${{ runner.os }}-junit- # key for restoring the cache if no exact match is found
	# In this step, we are downloading the latest artifact from the build job and storing it in the container
	- run: \|
	# Download the latest tests results artifact number from the GitHub API using jq to parse the JSON response
	curl \
	-H "Accept: application/vnd.github+json" \
	-H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}"\
	-H "X-GitHub-Api-Version: 2022-11-28" \
	https://api.github.com/repos/${{ github.repository }}/actions/artifacts \| jq -r '.artifacts \| sort_by(.created_at) \| .[] \| select(.name == "Test Results") \| .id' > artifacts_list.txt

	LATEST_ARTIFACT_NUMBER=$(cut -d: -f 2 artifacts_list.txt \| sort -n \| tail -n 1)

	curl \
	-H "Accept: application/vnd.github+json" \
	-H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \
	-H "X-GitHub-Api-Version: 2022-11-28" \
	-L -o my_artifact.zip \
	https://api.github.com/repos/${{ github.repository }}/actions/artifacts/"${LATEST_ARTIFACT_NUMBER}"/zip

	mkdir test_results
	unzip my_artifact.zip -d test_results 2> /dev/null \|\| true

	- name: Setup tmate session
	uses: mxschmitt/action-tmate@v3
	if: ${{ github.event_name == 'workflow_dispatch' && inputs.ssh_debug_enabled }}

	# split-tests action - splits the tests into x number of groups
	# based on the total number of github-hosted runners and junit previous test results by time and line count.
	# Link to the action - https://github.com/marketplace/actions/split-tests
	- uses: chaosaffe/[email protected]
	id: split-tests
	name: Split tests
	with:
	glob: src/test///*.java # glob pattern to match the test files
	split-total: ${{ env.total-runners }} # total number of github-hosted runners
	split-index: ${{ matrix.runner-index }} # current runner index
	junit-path: test_results/*xml # path to the junit test results with wildcards to match all the files
	line-count: true # split the tests based on the junit test results by line count
	# run the tests in parallel looping through the test-suite output from the split-tests action
	- run: 'echo "This runner will execute the following tests: ${{ steps.split-tests.outputs.test-suite }}"'
	- run: \|
	LIST="${{ steps.split-tests.outputs.test-suite }}"
	for file in $LIST
	do
	# sleep for 10 seconds to avoid timeout errors
	sleep 10
	mvn -Dtest=$(basename $file \| sed -e "s/.java/,/" \| tr -d '\r\n') -e test -Dspring.datasource.url=${{ secrets.LIQUIBASE_COMMAND_URL }} -Dspring.datasource.username=${{ secrets.LIQUIBASE_COMMAND_USERNAME }} -Dspring.datasource.password=${{ secrets.LIQUIBASE_COMMAND_PASSWORD }} -Dspring.liquibase.change-log=classpath:db/changelog/changelog_version-3.3.xml -Dserver.port=8086 -Dspring.redis.host=redis -Dspring.redis.port=6379 -Dspring.redis.mode=standalone
	done

	- uses: actions/upload-artifact@v3 # upload the test results as an artifact
	with:
	name: Test Results
	path: ./target/surefire-reports # path to the test results
	retention-days: 90 # retention period for the artifact in days. Link to the documentation - https://docs.github.com/en/actions/guides/storing-workflow-data-as-artifacts#about-workflow-artifact-retention

	publish-test-results:
	needs: unit-parallel-tests
	runs-on: ubuntu-latest
	steps:
	- name: Checkout repository
	uses: actions/checkout@v3

	- name: Download test results
	uses: actions/download-artifact@v2
	with:
	name: Test Results
	path: test_results

	- name: Publish Test Results
	uses: dorny/[email protected]
	if: success() \|\| failure()
	with:
	reporter: java-junit
	name: JUnit Test Results
	path: test_results/*.xml

	build-and-publish-docker-image: # job to build the docker image and publish it to the GitHub Container Registry
	runs-on: ubuntu-latest # using the latest ubuntu runner
	outputs:
	image_tag: ghcr.io/${{ github.repository }}:${{ github.run_number }} # output the image tag to be used in the build-and-publish-docker-image job
	needs: [build, unit-parallel-tests] # this job needs build and unit-parallel-tests jobs as a requirement to run
	if: github.ref == 'refs/heads/main' # run this job only when the branch is main branch and not on pull requests or other branches - https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#github-context
	# permissions for write acces to the packages and id-token and push access to the repository to create the container registry token
	permissions:
	packages: write
	id-token: write
	contents: write

	# steps to run the unit-parallel-tests job are as follows:
	# 1. checkout the repository
	# 2. download the jar artifact from the build job
	# 3. use the docker layer caching action to speed up the docker image build process
	# 4. build the docker image
	# 5. log in to the GitHub Container Registry
	# 6. push the docker image to the GitHub Container Registry
	steps:
	- name: Checkout repository
	uses: actions/checkout@v3
	- uses: actions/download-artifact@v1
	with:
	name: jar-artifact
	path: target/

	# build the docker image using the Dockerfile in the root of the repository
	# and tag it with the current run number from the github action workflow run
	- name: Log in to the GH Container Registry
	uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 # using the docker login action from the github marketplace - github.com/marketplace/actions/docker-login
	with:
	registry: ${{ env.REGISTRY }} # using the registry environment variable
	username: ${{ github.actor }} # using the github.actor context
	password: ${{ secrets.GITHUB_TOKEN }} # using the GITHUB_TOKEN secret

	- name: Build and push Docker image
	id: build_image
	uses: docker/build-push-action@v4 # using the docker build and push action from the github marketplace - github.com/marketplace/actions/build-and-push-docker-images
	with:
	context: . # using the current directory as the context
	push: true # push the docker image to the registry
	tags: \|
	ghcr.io/${{ github.repository }}:${{ github.run_number }}
	ghcr.io/${{ github.repository }}:latest
	cache-from: type=registry,ref=ghcr.io/${{ github.repository }}:latest # use the docker layer caching to speed up the docker image build process
	cache-to: type=inline

	runner-indexes: # job to generate the runner indexes for the unit-parallel-tests job
	runs-on: ubuntu-latest
	name: Generate runner indexes
	outputs:
	json: ${{ steps.generate-index-list.outputs.json }} # output the json with the runner indexes
	steps:
	- id: generate-index-list # generate the runner indexes and save them to the json file
	run: \|
	MAX_INDEX=$((${{ env.total-runners }}-1)) # calculate the max index
	INDEX_LIST=$(seq 0 ${MAX_INDEX}) # generate the list of indexes
	INDEX_JSON=$(jq --null-input --compact-output '. \|= [inputs]' <<< ${INDEX_LIST}) # convert the list to the json
	echo "json=${INDEX_JSON}" >> $GITHUB_OUTPUT # save the json to the GITHUB_OUTPUT environment variable

	deploy:
	needs: [build-and-publish-docker-image] # this job needs build-and-publish-docker-image job as a requirement to run
	uses: ./.github/workflows/cd.yml
	with:
	# with tag from the build-and-publish-docker-image job in the output_tags step
	image_tag: "${{ needs.build-and-publish-docker-image.outputs.image_tag }}"
	debug: "${{ github.event.inputs.debug_deployment }}"
	secrets: inherit

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

CI #284

Workflow file

CI #284

Jobs

Run details

Workflow file for this run