ci.yml

name: Tinybird CI

on:
  workflow_call:
    inputs:
      data_project_dir:
        description: "relative path of the folder containing the data project"
        required: false
        type: string
        default: .
    secrets:
      admin_token:
        required: true
      tb_host:
        required: true

jobs:
  ci_branching:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: ${{ inputs.data_project_dir }}
    steps:
      - uses: actions/checkout@master
        with:
          fetch-depth: 300
          ref: ${{ github.event.pull_request.head.sha }}
      - uses: actions/setup-python@v3
        with:
          python-version: "3.8"
          architecture: "x64"

      - name: Validate input
        run: |
          [[ "${{ secrets.admin_token }}" ]] || { echo "Set ADMIN_TOKEN variable"; exit 1; }

      - name: Set environment variables
        run: |
          _ENV_FLAGS="${ENV_FLAGS:=--last-partition --wait}"
          _NORMALIZED_ENV_NAME=$(echo $DATA_PROJECT_DIR | rev | cut -d "/" -f 1 | rev | tr '.-' '_')
          GIT_BRANCH=${GITHUB_HEAD_REF}
          source .tinyenv
          echo "GIT_BRANCH=$GIT_BRANCH" >> $GITHUB_ENV
      - name: Install Tinybird CLI
        run: pip install tinybird-cli

      - name: Tinybird version
        run: tb --version       

      - name: Check all the data files syntax
        run: tb check  

      - name: Create new test Environment with data
        run: |
          tb \
          --host ${{ secrets.tb_host }} \
          --token ${{ secrets.admin_token }} \
          env create tmp_ci_${_NORMALIZED_ENV_NAME}_${GITHUB_RUN_ID} \
          ${_ENV_FLAGS}

      - name: List changes with Main Environment
        run: tb diff --main --no-verbose

      - name: Push changes to the test Environment
        run: |
          source .tinyenv
          CI_DEPLOY_FILE=./deploy/${VERSION}/ci-deploy.sh
          if [ -f "$CI_DEPLOY_FILE" ]; then
            ./deploy/${VERSION}/ci-deploy.sh
          else
            tb deploy --populate --fixtures --wait
          fi

      - name: List changes with test Environment (should be empty)
        run: tb diff

      - name: Run fixture tests
        run: |
          if [ -f ./scripts/exec_test.sh ]; then
            ./scripts/exec_test.sh
          fi

      - name: Run data quality tests
        run: |
          tb test run -v

      - name: Get regression labels
        id: regression_labels
        uses: SamirMarin/get-labels-action@v0
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          label_key: regression

      - name: Run pipe regression tests
        run: |
          echo ${{ steps.regression_labels.outputs.labels }}
          REGRESSION_LABELS=$(echo "${{ steps.regression_labels.outputs.labels }}" | awk -F, '{for (i=1; i<=NF; i++) if ($i ~ /^--/) print $i}' ORS=',' | sed 's/,$//')
          echo ${REGRESSION_LABELS}
          tb env regression-tests coverage --wait $(echo ${REGRESSION_LABELS} | tr , ' ')
  cleanup:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: ${{ inputs.data_project_dir }}
    if: ${{ always() }}
    needs: [ci_branching]
    steps:
      - uses: actions/checkout@master
      - uses: actions/setup-python@v3
        with:
          python-version: "3.8"
          architecture: "x64"

      - name: Validate input
        run: |
          [[ "${{ secrets.admin_token }}" ]] || { echo "Set ADMIN_TOKEN variable"; exit 1; }

      - name: Install Tinybird CLI
        run: pip install tinybird-cli

      - name: Tinybird version
        run: tb --version       

      - name: Drop test Environment
        run: |
          tb \
          --host ${{ secrets.tb_host }} \
          --token ${{ secrets.admin_token }} \
          env rm tmp_ci_${_NORMALIZED_ENV_NAME}_${GITHUB_RUN_ID} \
          --yes