build-and-test.yaml

name: Build and Test
run-name: Building and testing
on:
  workflow_dispatch:
    inputs:
      release:
        required: false
        type: boolean
        default: false
      release-to-pypi:
        required: false
        type: boolean
        default: false
  push:

jobs:
  params:
    runs-on: ubuntu-latest
    outputs:
      ocm_repository: ${{ steps.params.outputs.ocm_repository }}
      oci_repository: ${{ steps.params.outputs.oci_repository }}
      oci_platforms: ${{ steps.params.outputs.oci_platforms }}
      gcp_project: ${{ steps.params.outputs.gcp_project }}
      gcp_service_account: ${{ steps.params.outputs.service_account }}
      workload_identity_provider: ${{ steps.params.outputs.idp }}
    steps:
      - name: params
        id: params
        run: |
          repo_base=europe-docker.pkg.dev/gardener-project
          snapshots_repo="${repo_base}/snapshots"
          releases_repo="${repo_base}/releases"

          if ${{ inputs.release || false }}; then
            ocm_repository=${releases_repo}
            oci_repository=${releases_repo}
          else
            ocm_repository=${snapshots_repo}
            oci_repository=${snapshots_repo}
          fi

          echo "ocm_repository=${ocm_repository}" >> "${GITHUB_OUTPUT}"
          echo "oci_repository=${oci_repository}" >> "${GITHUB_OUTPUT}"
          echo "oci_platforms=linux/amd64,linux/arm64" >> "${GITHUB_OUTPUT}"

          echo "gcp_project=gardener-project" >> "${GITHUB_OUTPUT}"
          echo "service_account=ocm-ci-opensource@gardener-project.iam.gserviceaccount.com" \
            >> "${GITHUB_OUTPUT}"
          echo "idp=projects/694386720375/locations/global/workloadIdentityPools/github-actions/providers/github-actions-provider" \
            >> "${GITHUB_OUTPUT}"

  version:
    runs-on: ubuntu-latest
    outputs:
      effective_version: ${{ steps.version.outputs.effective_version }}
      next_version: ${{ steps.version.outputs.next_version }}
      repo_version: ${{ steps.version.outputs.repo_version }}
      setuptools_version: ${{ steps.version.outputs.setuptools_version }}
    steps:
    - uses: actions/checkout@v4
    - name: calculate-effective-version
      id: version
      run: |
        src_version=$(.ci/read-version)
        commit=${{ github.sha }}
        echo "commit-digest: ${commit}"
        major="$(echo ${src_version} | cut -d. -f1)"
        minor="$(echo ${src_version} | cut -d. -f2)"
        patch="$(echo ${src_version} | cut -d. -f3 | cut -d- -f1)"

        if ${{ inputs.release || false }}; then
          effective_version=${major}.${minor}.${patch}
          setuptools_version=${effective_version}
          # hardcode to bumping "minor" for now
          next_minor=$(( $minor + 1 ))
          next_version="${major}.${next_minor}.${patch}-dev"
          echo "next_version=${next_version}" >> "${GITHUB_OUTPUT}"
          echo "next_version=${next_version}"
        else
          effective_version=${major}.${minor}.${patch}-${commit}
          setuptools_version=${src_version}
        fi
        echo "effective-version: ${effective_version}"
        echo "effective_version=${effective_version}" >> "${GITHUB_OUTPUT}"
        echo "repo_version=${src_version}" >> "${GITHUB_OUTPUT}"
        echo "setuptools_version=${setuptools_version}" >> "${GITHUB_OUTPUT}"

  package:
    runs-on: ubuntu-latest
    environment: build
    outputs:
      ocm_resources: ${{ steps.package.outputs.ocm_resources }}
    needs: version
    container:
      image: python:alpine
    steps:
    - name: Install git, setuptools, node (for upload-artifact)
      run: |
        apk add \
          bash \
          file \
          git \
          nodejs \
          xz
        pip3 install --root-user-action ignore \
          setuptools \
          pyyaml
    - uses: actions/checkout@v4
    - uses: actions/setup-node@v4 # required by upload-artifact / `act`
      # see: https://github.com/nektos/act/issues/973
    - name: create distribution package
      id: package
      run: |
        set -eu
        version=${{ needs.version.outputs.setuptools_version }}
        echo "version: ${version}"
        echo "${version}" | .ci/write-version
        pkg_dir=dist
        mkdir -p ${pkg_dir}
        pkg_dir="$(readlink -f dist)"
        echo "pkgdir: ${pkg_dir}"
        for path in \
          setup.py \
          setup.oci.py \
          setup.whd.py \
        ; do
         echo "building distribution package from ${path}"
         python3 ${path} \
            bdist_wheel \
            --dist-dir ${pkg_dir}
         set -x
         rm -rf build
         set +x
        done

        # special-case: cli-package (need to chdir in order to not confuse setuptools)
        (
          cd cli
          python3 setup.py \
            bdist_wheel \
            --dist-dir ${pkg_dir}
        )

        echo "Built packages"
        ls "${pkg_dir}"
        blobs_dir="${pkg_dir}/blobs.d"
        mkdir ${blobs_dir}
        resources_file=resources.yaml
        for package in gardener-oci gardener-cicd-whd gardener-cicd-cli gardener-cicd-libs; do
          path="$(echo $package | tr - _)-*"
          prefix=dist
          access_type='localBlob'
          outf="${pkg_dir}/${package}.tar.gz"
          tar cJf ${outf} -C ${pkg_dir} $(cd ${pkg_dir}; ls ${path})
          mimetype=$(file -i ${outf} | cut -d: -f2 | cut -d' ' -f2-)
          leng=$(stat -c"%s" ${outf})
          digest="$(sha256sum ${outf} | cut -d' ' -f1)"
          echo "\
        - name: ${package}
          version: ${version}
          type: ${mimetype}
          relation: local
          access:
            type: localBlob
            localReference: sha256:${digest}
            size: ${leng}
            mediaType: ${mimetype}" \
          >> ${resources_file}
          mv ${outf} ${blobs_dir}/${digest}
        done
        cp "${resources_file}" dist/ocm_resources.yaml
        echo "ocm_resources=dist/ocm_resources.yaml" >> "${GITHUB_OUTPUT}"
        find "${pkg_dir}"
    - uses: actions/upload-artifact@v4
      with:
        name: distribution-packages
        path: dist/

  component_descriptor:
    name: OCM + Release (only on manual triggering)
    runs-on: ubuntu-latest
    container:
      image: python:alpine
    permissions:
      contents: write
      id-token: write
    needs:
      - version
      - params
      - package
      - images
      - lint
      - unittests
      - documentation
    outputs:
      release_commit_digest: ${{  steps.releasecommit.commitdigest }}
      ocm_repository: ${{ steps.params.outputs.ocm_repository }}
    steps:
      - name: Install Packages
        run: |
          apk add --no-cache \
            bash \
            git
          git config --global --add safe.directory /__w/cc-utils/cc-utils
      - uses: actions/checkout@v4
      - name: Retrieve Distribution Packages
        uses: actions/download-artifact@v4
        with:
          name: distribution-packages
          path: /tmp/dist
      - name: Retrieve Linting Logs
        uses: actions/download-artifact@v4
        with:
          name: linting-logs # targetpath: bandit.tar.gz
          path: /tmp/linting-logs
      - name: Retrieve Documentation
        uses: actions/download-artifact@v4
        with:
          name: documentation
          path: /tmp/documentation-out.d

      - name: Google-Auth
        id: auth
        uses: google-github-actions/auth@v2
        with:
          token_format: access_token
          project_id: ${{ needs.params.outputs.gcp_project }}
          service_account: ${{ needs.params.outputs.gcp_service_account }}
          workload_identity_provider: ${{ needs.params.outputs.workload_identity_provider }}
      - name: Create Release and Bump-Commits
        id: releasecommit
        if: ${{ inputs.release }}
        run: |
          echo "gha-creds*" >> .git/info/exclude
          git config --global --add safe.directory /__w/cc-utils/cc-utils
          git config user.name 'Gardener-CICD Bot'
          git config user.email 'gardener.ci.user@gmail.com'
          effective_version=${{ needs.version.outputs.effective_version }}
          echo "${effective_version}" | .ci/write-version
          git add .
          git commit -m "Release ${effective_version}"
          git show
          commit_digest=$(git rev-parse @)
          echo "release_commit_digest=${commit_digest}" >> "${GITHUB_OUTPUT}"
          tgt_ref="refs/tags/${effective_version}"
          echo "pushing release-commit ${commit_digest} to ${tgt_ref}"
          git push origin "@:${tgt_ref}"
          next_version=${{ needs.version.outputs.next_version }}
          echo "next version: ${next_version}"
          git reset --hard @~
          echo "${next_version}" | .ci/write-version
          git add .
          git commit -m"Prepare next Dev-Cycle"
          git pull --rebase
          git push origin
      - name: component-descriptor
        run: |
          set -eu
          pip3 install --no-cache \
            aiohttp \
            dacite \
            deprecated \
            pyaml \
            python-dateutil \
            requests \
            www-authenticate \
          &>/dev/null
          python -c "import oci"
          version=${{ needs.version.outputs.effective_version }}
          ocm_repo=${{ needs.params.outputs.ocm_repository }}
          echo "generating component-descriptor"
          python3 -m ocm create \
            --name github.com/${{ github.repository }} \
            --version ${version} \
            --ocm-repo ${ocm_repo} \
            --provider sap-se \
            --label '{
                "name": "cloud.gardener.cnudie/responsibles",
                "value": [
                  {
                    "type": "githubTeam",
                    "teamname": "gardener/ci-maintainers",
                    "github_hostname": "github.com"
                  }
                ]
            }' \
            > component-descriptor.yaml

          if ${{ inputs.release || false }}; then
            commit_digest=${{ steps.releasecommit.outputs.release_commit_digest || '' }}
          else
            commit_digest=${{ github.sha }}
          fi

          echo "adding main source"
          echo "\
          name: main-source
          version: ${version}
          type: git
          labels:
            - name: cloud.gardener/cicd/source
              value:
                repository-classification: main
          access:
            type: github
            repoUrl: github.com/${{ github.repository }}
            version: ${version}
            commit: ${commit_digest}
            ref: ${{ github.ref }}
          " \
          | python3 -m ocm append source \
            --file component-descriptor.yaml

          echo "adding resources"
          ocm_resources=${{ needs.package.outputs.ocm_resources }}
          echo "ocm-resources-file: /tmp/${ocm_resources}"
          cat "/tmp/${ocm_resources}" | \
          python3 -m ocm append resource \
            --file component-descriptor.yaml

          echo "adding linting-evidence resource"
          linting_evidence=/tmp/linting-logs/bandit.tar.gz
          linting_digest=$(sha256sum ${linting_evidence} | cut -d' ' -f1)
          cp ${linting_evidence} /tmp/dist/blobs.d/${linting_digest}

          cat << EOF > linting_evidence.ocm-resource
          name: sast-linting-evidence
          version: ${version}
          type: application/gzip
          relation: local
          access:
            type: localBlob
            localReference: sha256:${linting_digest}
            size: $(stat -c"%s" ${linting_evidence})
          labels:
            - name: gardener.cloud/purposes
              value:
                - lint
                - sast
                - pybandit
            - name: gardener.cloud/comment
              value: |
                we use bandit (linter) for SAST scans
                see: https://bandit.readthedocs.io/en/latest/
          EOF
          cat linting_evidence.ocm-resource | \
          python3 -m ocm append resource \
            --file component-descriptor.yaml

          echo "${{ needs.images.outputs.ocm_resources }}" | base64 -d > oci_ocm_resources.yaml

          cat oci_ocm_resources.yaml | \
          python3 -m ocm append resource \
            --file component-descriptor.yaml

          echo "component-descriptor to be uploaded:"
          cat component-descriptor.yaml

          # from google-auth
          username=oauth2accesstoken
          password=${{ steps.auth.outputs.access_token }}
          token=$(echo -n ${username}:${password} | base64 -w0)
          mkdir $HOME/.docker
          docker_cfg=$HOME/.docker/config.json
          registry_hostname=$(echo ${{ needs.params.outputs.ocm_repository }} | cut -d/ -f1)
          cat << EOF > ${docker_cfg}
          {
            "auths": {
              "${registry_hostname}": {
                "auth": "${token}"
              }
            }
          }
          EOF

          echo "uploading component-descriptor"
          python -m ocm upload \
            --file component-descriptor.yaml \
            --blobs-dir /tmp/dist/blobs.d
      - name: Publish Documentation
        if: ${{ inputs.release }}
        run: |
          release_commit_digest=${{ steps.releasecommit.outputs.release_commit_digest || '' }}
          git fetch origin gh-pages
          git checkout gh-pages
          git clean -dfx
          git status
          echo "let's hope our worktree is clean"
          tar c -C /tmp/documentation-out.d . | tar x -C.
          git status
          if [ -z "$(git status --porcelain)" ]; then
            echo "no changes in documentation - no need to update documentation"
            exit 0
          fi
          git add -A
          git commit -m "update documentation"
          git push origin refs/heads/gh-pages

  lint:
    runs-on: ubuntu-latest
    needs:
      - package
    container:
      image: python:alpine
    steps:
    - name: install git
      run: |
        apk add --no-cache git
    - uses: actions/checkout@v4
    - name: Retrieve Distribution Packages
      uses: actions/download-artifact@v4
      with:
        name: distribution-packages
        path: /tmp/dist
    - name: lint
      run: |
        echo "install dependencies for python-packages"
        if ! apk add --no-cache $(cat gardener-cicd-libs.apk-packages) >/tmp/apk.log; then
          echo "error while trying to install apk-packages:"
          cat /tmp/apk.log
          exit 1
        fi
        echo "installing linters"
        if ! pip3 install --upgrade --break-system-packages \
          --find-links /tmp/dist \
          gardener-cicd-libs \
          gardener-cicd-cli \
          gardener-oci \
          bandit \
          flake8 \
          setuptools \
          pylama \
          pylint \
          > /tmp/pip3-install.log; then
          echo "error while trying to install packages:"
          cat /tmp/pip3-install.log
        fi

        echo "running linters"
        bandit_logfile=bandit.log
        bandit_evidence=bandit.tar.gz

        bandit_extra_args="-f txt -o ${bandit_logfile}" \
          .ci/lint

        # pass bandit.log + used cfg (pyproject.toml) as evidence
        tar czf $bandit_evidence $bandit_logfile pyproject.toml

    - uses: actions/upload-artifact@v4
      with:
        name: linting-logs
        path: bandit.tar.gz

  unittests:
    needs:
      - package
    permissions:
      contents: read
    runs-on: ubuntu-latest
    container:
      image: python:alpine
    steps:
    - uses: actions/checkout@v4
    - name: Retrieve Distribution Packages
      uses: actions/download-artifact@v4
      with:
        name: distribution-packages
        path: /tmp/dist
    - name: run-tests
      run: |
        set -eu
        echo "install dependencies for python-packages"
        apk add --no-cache $(cat gardener-cicd-libs.apk-packages)
        echo "install packages"
        if ! pip3 install --break-system-packages \
          --find-links /tmp/dist \
          gardener-cicd-libs \
          gardener-cicd-cli \
          gardener-oci \
          pytest \
          setuptools \
            > /tmp/pip3-install.log; then
          echo "error while trying to install packages:"
          cat /tmp/pip3-install.log
        fi

        pip3 list

        echo "running tests"
        mkdir /tmp/fake-cfg.d
        touch /tmp/fake-cfg.d/config_types.yaml
        export CC_CONFIG_DIR=/tmp/fake-cfg.d
        .ci/test

  pypi:
    if: ${{ inputs.release-to-pypi }}
    runs-on: ubuntu-latest
    name: Publish to PYPI
    needs:
      - version
      - package
      - params
      - lint
      - unittests
    permissions:
      contents: read
      id-token: write
    steps:
      - name: Retrieve Distribution Packages
        uses: actions/download-artifact@v4
        with:
          name: distribution-packages
          path: /tmp/dist
      - name: prepare build-filesystem
        id: prepare
        run: |
          cp -r /tmp/dist .
          ls -lta dist/
          rm -rf dist/blobs.d dist/ocm_resources.yaml
          ls -lta dist/
      - name: publish to pypi
        uses: pypa/gh-action-pypi-publish@release/v1

  images:
    name: Build OCI Images
    needs:
      - version
      - package
      - params
    outputs:
      oci_image_ref: ${{ steps.prepare.outputs.oci_image_ref }}
      ocm_resources: ${{ steps.prepare.outputs.ocm_resources }}
    runs-on: ubuntu-latest
    environment: build
    permissions:
      contents: read
      id-token: write
    steps:
      - uses: actions/checkout@v4
      - name: Setup Docker-Buildx
        uses: docker/setup-buildx-action@v3
      - name: Google-Auth
        id: auth
        uses: google-github-actions/auth@v2
        with:
          token_format: access_token
          project_id: ${{ needs.params.outputs.gcp_project }}
          service_account: ${{ needs.params.outputs.gcp_service_account }}
          workload_identity_provider: ${{ needs.params.outputs.workload_identity_provider }}
      - name: docker-auth
        id: docker-auth
        uses: docker/login-action@v3
        with:
          username: oauth2accesstoken
          password: ${{ steps.auth.outputs.access_token }}
          registry: europe-docker.pkg.dev
      - name: Retrieve Distribution Packages
        uses: actions/download-artifact@v4
        with:
          name: distribution-packages
          path: /tmp/dist

      - name: prepare build-filesystem
        id: prepare
        run: |
          cp -r /tmp/dist .
          ls -lta

          setuptools_version=${{ needs.version.outputs.setuptools_version }}
          # workaround: set repository-version to setuptools-version so installation of
          #             packages will succeed
          echo "${setuptools_version}" | .ci/write-version
          oci_repo=${{ needs.params.outputs.oci_repository }}
          image_tag=${{ needs.version.outputs.effective_version }}
          image_ref=${oci_repo}/cicd/job-image:${image_tag}

          echo "oci_image_ref=${image_ref}" >> ${GITHUB_OUTPUT}
          cat << EOF > ocm_resources.yaml
          name: job-image
          version: ${image_tag}
          type: ociImage
          access:
            type: ociRegistry
            imageReference: ${image_ref}
          relation: local
          labels:
            - name: cloud.gardener.cnudie/dso/scanning-hints/package-versions
              value:
                - name: containerd
                  version: v1.6.15
            - name: gardener.cloud/cve-categorisation
              value:
                authentication_enforced: true
                availability_requirement: low
                confidentiality_requirement: high
                integrity_requirement: high
                network_exposure: protected
                user_interaction: gardener-operator
          EOF
          echo "ocm_resources=$(cat ocm_resources.yaml | base64 -w0)" >> ${GITHUB_OUTPUT}

      - name: Build OCI Image
        uses: docker/build-push-action@v6
        with:
          push: true
          platforms: ${{ needs.params.outputs.oci_platforms }}
          tags: ${{ steps.prepare.outputs.oci_image_ref }}
          context: . # pass modified path rather than clean checkout

  documentation:
    name: Generate Documentation
    needs:
      - images
    runs-on: ubuntu-latest
    environment: build
    permissions:
      contents: read
    steps:
    - uses: actions/checkout@v4
    - name: Setup Docker-Buildx
      uses: docker/setup-buildx-action@v3
    - name: Generate Documentation
      run: |
        image_ref=${{ needs.images.outputs.oci_image_ref }}
        mkdir documentation-out.d
        docker run -v$PWD:/src \
          -e GH_PAGES_PATH=/src/documentation-out.d \
          ${image_ref} \
          /src/.ci/generate_documentation
        ls documentation-out.d
    - uses: actions/upload-artifact@v4
      with:
        name: documentation
        path: documentation-out.d