ci.yml

name: CI

on:
  push:
    tags:
      - '*'
    branches:
      - master
      - develop
  pull_request:

jobs:
  test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - name: Set up JDK 11
        uses: actions/setup-java@v1
        with:
          java-version: 11
      - name: Install sbt
        uses: sbt/setup-sbt@v1
      - name: Check Scala formatting
        if: ${{ always() }}
        run: sbt scalafmtCheckAll scalafmtSbtCheck
      - name: Run tests
        run: sbt coverage test coverageReport
      - name: Aggregate coverage data
        if: ${{ always() }}
        run: sbt coverageAggregate
      - name: Submit coveralls data
        if: ${{ always() }}
        run: sbt coveralls
        env:
          COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }}

  publish_docker:
    needs: test
    if: startsWith(github.ref, 'refs/tags/')
    runs-on: ubuntu-latest
    strategy:
      matrix:
        app:
          - redshiftLoader
          - snowflakeLoader
          - databricksLoader
          - transformerKinesis
          - transformerPubsub
          - transformerKafka
    steps:
      - name: Checkout Github
        uses: actions/checkout@v2
      - uses: coursier/cache-action@v6
      - name: Set up JDK 11 for loader and streaming transformer
        uses: actions/setup-java@v1
        with:
          java-version: 11
      - name: Install sbt
        uses: sbt/setup-sbt@v1
      - name: Get the latest Databricks JDBC driver
        run: |
          curl https://databricks-bi-artifacts.s3.us-east-2.amazonaws.com/simbaspark-drivers/jdbc/2.6.40/DatabricksJDBC42-2.6.40.1070.zip --output DatabricksJDBC42.jar.zip
          unzip DatabricksJDBC42.jar.zip
          cp ./*/DatabricksJDBC42.jar . # 2.6.34 download changes directory structure - grab the jar from nested directory (which has entropy in its name)
      - name: Docker login
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}
      - name: Get current version
        id: ver
        run: echo "::set-output name=tag::${GITHUB_REF#refs/tags/}"
      - name: Get app package name
        id: packageName
        run: |
          export PACKAGE_NAME=$(sbt "project ${{ matrix.app }}" dockerAlias -Dsbt.log.noformat=true | sed -n '/\[info\]/ s/\[info\] //p' | tail -1 | tr -d '\n' | cut -d":" -f1)
          echo "::set-output name=package_name::$PACKAGE_NAME"
      - name: Get app base directory
        id: baseDirectory
        run: |
          export BASE_DIRECTORY=$(sbt "project ${{ matrix.app }}" baseDirectory -Dsbt.log.noformat=true | sed -n '/\[info\]/ s/\[info\] //p' | tail -1 | tr -d '\n')
          echo "::set-output name=directory::$BASE_DIRECTORY"
      - name: Get app base directory (distroless)
        id: baseDirectoryDistroless
        run: |
          export BASE_DIRECTORY_DISTROLESS=$(sbt "project ${{ matrix.app }}Distroless" baseDirectory -Dsbt.log.noformat=true | sed -n '/\[info\]/ s/\[info\] //p' | tail -1 | tr -d '\n')
          echo "::set-output name=directory::$BASE_DIRECTORY_DISTROLESS"
      - name: Stage the Docker build
        run: sbt "project ${{ matrix.app}}" docker:stage
      - name: Stage the Docker distroless build
        run: sbt "project ${{ matrix.app }}Distroless" docker:stage
      - name: Docker metadata
        id: meta
        uses: docker/metadata-action@v3
        with:
          images: ${{ steps.packageName.outputs.package_name }}
          tags: |
            type=raw,value=latest,enable=${{ !contains(steps.ver.outputs.tag, 'rc') }}
            type=raw,value=latest-focal,enable=${{ !contains(steps.ver.outputs.tag, 'rc') }}
            type=raw,value=${{ steps.ver.outputs.tag }}
            type=raw,value=${{ steps.ver.outputs.tag }}-focal
          flavor: |
            latest=false
      - name: Docker metadata distroless
        id: distroless-meta
        uses: docker/metadata-action@v3
        with:
          images: ${{ steps.packageName.outputs.package_name }}
          tags: |
            type=raw,value=latest-distroless,enable=${{ !contains(steps.ver.outputs.tag, 'rc') }}
            type=raw,value=${{ steps.ver.outputs.tag }}-distroless
          flavor: |
            latest=false
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v1
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1
      - name: Push image
        uses: docker/build-push-action@v2
        with:
          context: ${{ steps.baseDirectory.outputs.directory }}/target/docker/stage
          file: ${{ steps.baseDirectory.outputs.directory }}/target/docker/stage/Dockerfile
          platforms: linux/amd64,linux/arm64/v8
          tags: ${{ steps.meta.outputs.tags }}
          push: true
      - name: Push distroless image
        uses: docker/build-push-action@v2
        with:
          context: ${{ steps.baseDirectoryDistroless.outputs.directory }}/target/docker/stage
          file: ${{ steps.baseDirectoryDistroless.outputs.directory }}/target/docker/stage/Dockerfile
          platforms: linux/amd64,linux/arm64/v8
          tags: ${{ steps.distroless-meta.outputs.tags }}
          push: true
      - name: Build local distroless image, which is needed to run Snyk
        if: ${{ !contains(github.ref_name, 'rc') }}
        run: sbt "project ${{ matrix.app }}Distroless" docker:publishLocal
      - name: Run Snyk to check for vulnerabilities
        uses: snyk/actions/docker@master
        if: ${{ !contains(github.ref_name, 'rc') }}
        with:
          image: "${{ steps.packageName.outputs.package_name }}:${{ github.ref_name }}-distroless"
          args: "--app-vulns --org=data-processing-new"
          command: monitor
        env:
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}