Merge pull request #1401 from maykinmedia/issue/celery-concurrency #7619
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - develop | |
| tags: | |
| - '**' | |
| pull_request: | |
| workflow_dispatch: | |
| env: | |
| IMAGE_NAME: maykinmedia/open-inwoner | |
| jobs: | |
| # | |
| # Unit and integration tests | |
| # | |
| tests: | |
| name: Run the Django test suite | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| test-type: [main, elastic, openklant] | |
| services: | |
| postgres: | |
| image: postgis/postgis:15-3.4 | |
| env: | |
| POSTGRES_HOST_AUTH_METHOD: trust | |
| ports: | |
| - 5432:5432 | |
| # needed because the postgres container does not provide a healthcheck | |
| options: | |
| --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 | |
| --name postgres | |
| elasticsearch: | |
| image: docker.elastic.co/elasticsearch/elasticsearch:7.9.2 | |
| env: | |
| discovery.type: single-node | |
| ES_JAVA_OPTS: '-Xms512m -Xmx512m' | |
| ports: | |
| - 9200:9200 | |
| - 9300:9300 | |
| redis: | |
| image: redis | |
| ports: | |
| - 6379:6379 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up backend environment | |
| uses: maykinmedia/[email protected] | |
| with: | |
| apt-packages: 'libxml2-dev libxmlsec1-dev libxmlsec1-openssl gettext postgresql-client libgdal-dev gdal-bin' | |
| python-version: '3.11' | |
| optimize-postgres: 'yes' | |
| pg-service: 'postgres' | |
| setup-node: 'yes' | |
| npm-ci-flags: '--legacy-peer-deps' | |
| - name: Run tests | |
| run: | | |
| if [ "${{ matrix.test-type }}" = "main" ]; then | |
| coverage run -p --concurrency=multiprocessing \ | |
| src/manage.py test src \ | |
| --parallel \ | |
| --exclude-tag=e2e \ | |
| --exclude-tag=elastic | |
| elif [ "${{ matrix.test-type }}" = "elastic" ]; then | |
| coverage run -p src/manage.py test src --tag=elastic --exclude-tag=e2e | |
| elif [ "${{ matrix.test-type }}" = "openklant" ]; then | |
| coverage run -m pytest --block-network --record-mode=none -vvv src/openklant2 | |
| else | |
| echo "Error: Unknown test type '${{ matrix.test-type }}'" | |
| exit 1 | |
| fi | |
| env: | |
| DJANGO_SETTINGS_MODULE: open_inwoner.conf.ci | |
| SECRET_KEY: dummy | |
| DB_USER: postgres | |
| DB_PASSWORD: '' | |
| - name: Persist coverage data files | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.test-type }} | |
| path: .coverage* | |
| include-hidden-files: true | |
| if-no-files-found: error | |
| upload_coverage: | |
| needs: tests | |
| name: Upload coverage information to codecov | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Set up Git repository | |
| uses: actions/checkout@v2 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Install coverage.py | |
| # We only need coverage, but we need to match the version that was used | |
| # to generate the coverage files. Grab it from the dependencies. | |
| run: grep "^coverage" requirements/ci.txt | xargs -r pip install | |
| - uses: actions/download-artifact@v4 | |
| with: | |
| path: coverages | |
| - name: Combine the coverage files | |
| run: | | |
| mv coverages/main/.coverage* . | |
| mv coverages/elastic/.coverage* . | |
| mv coverages/openklant/.coverage* . | |
| coverage combine | |
| - name: Publish coverage report | |
| uses: codecov/codecov-action@v3 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| # | |
| # End-to-end tests | |
| # | |
| e2etests: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| browser: | |
| - chromium | |
| - firefox | |
| - webkit | |
| - msedge | |
| name: End-to-end tests, ${{ matrix.browser }} | |
| services: | |
| postgres: | |
| image: postgis/postgis:15-3.4 | |
| env: | |
| POSTGRES_HOST_AUTH_METHOD: trust | |
| ports: | |
| - 5432:5432 | |
| # Needed because the postgres container does not provide a healthcheck | |
| options: | |
| --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 | |
| --name postgres | |
| elasticsearch: | |
| image: docker.elastic.co/elasticsearch/elasticsearch:7.9.2 | |
| env: | |
| discovery.type: single-node | |
| ES_JAVA_OPTS: '-Xms512m -Xmx512m' | |
| ports: | |
| - 9200:9200 | |
| - 9300:9300 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Set up backend environment | |
| uses: maykinmedia/setup-django-backend@v1 | |
| with: | |
| apt-packages: 'libxml2-dev libxmlsec1-dev libxmlsec1-openssl gettext postgresql-client libgdal-dev gdal-bin' | |
| python-version: '3.11' | |
| optimize-postgres: 'yes' | |
| pg-service: 'postgres' | |
| setup-node: 'yes' | |
| npm-ci-flags: '--legacy-peer-deps' | |
| # See https://playwright.dev/python/docs/ci#caching-browsers | |
| - name: Cache Playwright browser | |
| id: cache-browser | |
| uses: actions/cache@v3 | |
| with: | |
| path: /home/runner/.cache/ms-playwright | |
| key: ${{ runner.os }}-${{ matrix.browser }}-playwright-${{ hashFiles('requirements/ci.txt') }} | |
| - name: Install playwright deps | |
| run: playwright install --with-deps ${{ matrix.browser }} | |
| - name: Run testsuite | |
| run: | | |
| python src/manage.py collectstatic --noinput --link | |
| src/manage.py test src --tag=e2e | |
| env: | |
| DJANGO_SETTINGS_MODULE: open_inwoner.conf.ci | |
| SECRET_KEY: dummy | |
| DB_USER: postgres | |
| DB_PASSWORD: '' | |
| E2E_DRIVER: ${{ matrix.browser }} | |
| # | |
| # Docs | |
| # | |
| docs: | |
| name: Build and check documentation | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: 'true' | |
| - name: Set up backend environment | |
| uses: maykinmedia/setup-django-backend@v1 | |
| with: | |
| apt-packages: 'libxml2-dev libxmlsec1-dev libxmlsec1-openssl gettext libgdal-dev gdal-bin graphviz' | |
| python-version: '3.11' | |
| setup-node: 'no' | |
| - name: Build and test docs | |
| run: | | |
| export OPENSSL_CONF=$(pwd)/openssl.conf | |
| pytest check_sphinx.py -v --tb=auto | |
| working-directory: docs | |
| setup-config-docs: | |
| name: Check that documentation for configuration steps is up-to-date | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: 'true' | |
| - name: Set up backend environment | |
| uses: maykinmedia/setup-django-backend@v1 | |
| with: | |
| apt-packages: 'libxml2-dev libxmlsec1-dev libxmlsec1-openssl gettext libgdal-dev gdal-bin graphviz' | |
| python-version: '3.11' | |
| setup-node: 'no' | |
| - name: Build docs for django-setup-configuration steps | |
| run: | | |
| python src/manage.py generate_config_docs --dry-run | |
| # | |
| # Docker | |
| # | |
| docker: | |
| needs: tests | |
| name: Build (and push) Docker image | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Set tag | |
| id: vars | |
| run: | | |
| # Strip git ref prefix from version | |
| VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') | |
| # Strip "v" prefix from tag name (if present at all) | |
| [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') | |
| # Use Docker `latest` tag convention | |
| [ "$VERSION" == "develop" ] && VERSION=latest | |
| echo ::set-output name=tag::${VERSION} | |
| - name: Build the Docker image | |
| env: | |
| RELEASE_VERSION: ${{ steps.vars.outputs.tag }} | |
| run: docker build . --tag $IMAGE_NAME:$RELEASE_VERSION | |
| - name: Log into registry | |
| if: github.event_name == 'push' # exclude PRs | |
| run: echo "${{ secrets.DOCKER_TOKEN }}" | docker login -u ${{ secrets.DOCKER_USERNAME }} --password-stdin | |
| - name: Push the Docker image | |
| if: github.event_name == 'push' # exclude PRs | |
| env: | |
| RELEASE_VERSION: ${{ steps.vars.outputs.tag }} | |
| run: docker push $IMAGE_NAME:$RELEASE_VERSION |