build_wheels.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# To learn more about GitHub Actions in Apache Beam check the CI.md

name: Build python source distribution and wheels

on:
  schedule:
    - cron: '10 2 * * *'
  push:
    branches: ['master', 'release-*']
    tags: 'v*'
  pull_request:
    branches: ['master', 'release-*']
    tags: 'v*'
    paths: ['sdks/python/**', 'model/**', 'release/**']


env:
  GCP_PATH: "gs://${{ secrets.GCP_PYTHON_WHEELS_BUCKET }}/${GITHUB_REF##*/}/${GITHUB_SHA}-${GITHUB_RUN_ID}/"


jobs:

  check_gcp_variables:
    timeout-minutes: 5
    name: "Check GCP variables"
    runs-on: ubuntu-latest
    outputs:
      gcp-variables-set: ${{ steps.check_gcp_variables.outputs.gcp-variables-set }}
    steps:
      - uses: actions/checkout@v2
      - name: "Check are GCP variables set"
        run: "./scripts/ci/ci_check_are_gcp_variables_set.sh"
        id: check_gcp_variables
        env:
          GCP_SA_EMAIL: ${{ secrets.GCP_SA_EMAIL }}
          GCP_SA_KEY: ${{ secrets.GCP_SA_KEY }}
          GCP_PYTHON_WHEELS_BUCKET: ${{ secrets.GCP_PYTHON_WHEELS_BUCKET }}
          GCP_PROJECT_ID: "not-needed-here"
          GCP_REGION: "not-needed-here"
          GCP_TESTING_BUCKET: "not-needed-here"

  build_source:
    runs-on: ubuntu-latest
    name: Build python source distribution
    steps:
      - name: Checkout code
        uses: actions/checkout@v2
      - name: Install python
        uses: actions/setup-python@v2
        with:
          python-version: 3.7
      - name: Get build dependencies
        working-directory: ./sdks/python
        run: python -m pip install -r build-requirements.txt
      - name: Install wheels
        run: python -m pip install wheel
      - name: Build source
        working-directory: ./sdks/python
        run: python setup.py sdist --formats=zip
      - name: Add checksums
        working-directory: ./sdks/python/dist
        run: |
          file=$(ls | grep .zip | head -n 1)
          sha512sum $file > ${file}.sha512
      - name: Unzip source
        working-directory: ./sdks/python
        run: unzip dist/$(ls dist | grep .zip | head -n 1)
      - name: Rename source directory
        working-directory: ./sdks/python
        run: mv $(ls | grep apache-beam) apache-beam-source
      - name: Upload source as artifact
        uses: actions/upload-artifact@v2
        with:
          name: source
          path: sdks/python/apache-beam-source
      - name: Upload compressed sources as artifacts
        uses: actions/upload-artifact@v2
        with:
          name: source_zip
          path: sdks/python/dist

  prepare_gcs:
    name: Prepare GCS
    needs:
      - build_source
      - check_gcp_variables
    runs-on: ubuntu-latest
    if: needs.check_gcp_variables.outputs.gcp-variables-set == 'true' && github.event_name != 'pull_request'
    steps:
      - name: Authenticate on GCP
        uses: google-github-actions/setup-gcloud@master
        with:
          service_account_email: ${{ secrets.GCP_SA_EMAIL }}
          service_account_key: ${{ secrets.GCP_SA_KEY }}
      - name: Remove existing files on GCS bucket
        run: gsutil rm -r ${{ env.GCP_PATH }} || true

  upload_source_to_gcs:
    name: Upload python source distribution to GCS bucket
    needs:
      - prepare_gcs
      - check_gcp_variables
    runs-on: ubuntu-latest
    if: needs.check_gcp_variables.outputs.gcp-variables-set == 'true'
    steps:
      - name: Download compressed sources from artifacts
        uses: actions/download-artifact@v2
        with:
          name: source_zip
          path: source/
      - name: Authenticate on GCP
        uses: google-github-actions/setup-gcloud@master
        with:
          service_account_email: ${{ secrets.GCP_SA_EMAIL }}
          service_account_key: ${{ secrets.GCP_SA_KEY }}
      - name: Copy sources to GCS bucket
        run: gsutil cp -r -a public-read source/* ${{ env.GCP_PATH }}

  build_wheels:
    name: Build python wheels on ${{ matrix.os_python.os }}
    needs: build_source
    runs-on: ${{ matrix.os_python.os }}
    strategy:
      matrix:
        os_python: [
          {"os": "ubuntu-latest", "python": "cp36-* cp37-* cp38-*"},
          {"os": "macos-latest", "python": "cp36-* cp37-* cp38-*"},
          {"os": "windows-latest", "python": "cp36-* cp37-* cp38-*"},
        ]
    steps:
    - name: Download python source distribution from artifacts
      uses: actions/download-artifact@v2
      with:
        name: source
        path: apache-beam-source
    - name: Install Python
      uses: actions/setup-python@v2
      with:
        python-version: 3.7
    - name: Install cibuildwheel
      run: pip install cibuildwheel==1.4.2
    - name: Build wheel
      working-directory: apache-beam-source
      env:
        CIBW_BUILD: ${{ matrix.os_python.python }}
        CIBW_BEFORE_BUILD: pip install cython
      run: cibuildwheel --print-build-identifiers && cibuildwheel --output-dir wheelhouse
      shell: bash
    - name: install sha512sum on MacOS
      if: startsWith(matrix.os_python.os, 'macos')
      run: brew install coreutils
    - name: Add checksums
      working-directory: apache-beam-source/wheelhouse/
      run: |
        for file in *.whl; do
          sha512sum $file > ${file}.sha512
        done
      shell: bash
    - name: Upload wheels as artifacts
      uses: actions/upload-artifact@v2
      with:
        name: wheelhouse-${{ matrix.os_python.os }}
        path: apache-beam-source/wheelhouse/

  upload_wheels_to_gcs:
    name: Upload wheels to GCS bucket
    needs:
      - build_wheels
      - check_gcp_variables
    runs-on: ubuntu-latest
    if: needs.check_gcp_variables.outputs.gcp-variables-set == 'true' && github.event_name != 'pull_request'
    strategy:
      matrix:
        os : [ubuntu-latest, macos-latest, windows-latest]
    steps:
    - name: Download wheels from artifacts
      uses: actions/download-artifact@v2
      with:
        name: wheelhouse-${{ matrix.os }}
        path: wheelhouse/
    - name: Authenticate on GCP
      uses: google-github-actions/setup-gcloud@master
      with:
        service_account_email: ${{ secrets.GCP_SA_EMAIL }}
        service_account_key: ${{ secrets.GCP_SA_KEY }}
    - name: Copy wheels to GCS bucket
      run: gsutil cp -r -a public-read wheelhouse/* ${{ env.GCP_PATH }}
    - name: Create github action information file on GCS bucket
      run: |
        cat > github_action_info <<EOF
        GITHUB_WORKFLOW=$GITHUB_WORKFLOW
        GITHUB_RUN_ID=$GITHUB_RUN_ID
        GITHUB_RUN_NUMBER=$GITHUB_RUN_NUMBER
        GITHUB_ACTION=$GITHUB_ACTION
        GITHUB_ACTOR=$GITHUB_ACTOR
        GITHUB_REPOSITORY=$GITHUB_REPOSITORY
        GITHUB_EVENT_NAME=$GITHUB_EVENT_NAME
        GITHUB_SHA=$GITHUB_SHA
        GITHUB_REF=$GITHUB_REF
        # only for forked repositiories
        GITHUB_HEAD_REF=$GITHUB_HEAD_REF
        GITHUB_BASE_REF=$GITHUB_BASE_REF
        EOF
        echo $(cat github_action_info)
        gsutil cp -a public-read github_action_info ${{ env.GCP_PATH }}
    - name: Upload GitHub event file to GCS bucket
      run: gsutil cp -a public-read ${GITHUB_EVENT_PATH} ${{ env.GCP_PATH }}

  list_files_on_gcs:
    name: List files on Google Cloud Storage Bucket
    needs:
     - upload_wheels_to_gcs
     - check_gcp_variables
    runs-on: ubuntu-latest
    if: needs.check_gcp_variables.outputs.gcp-variables-set == 'true' && github.event_name != 'pull_request'
    steps:
    - name: Authenticate on GCP
      uses: google-github-actions/setup-gcloud@master
      with:
        service_account_email: ${{ secrets.GCP_SA_EMAIL }}
        service_account_key: ${{ secrets.GCP_SA_KEY }}
    - name: List file on Google Cloud Storage Bucket
      run: gsutil ls "${{ env.GCP_PATH }}*"

  tag_repo_nightly:
    name: Tag repo nightly
    needs:
      - build_source
      - build_wheels
    runs-on: ubuntu-latest
    timeout-minutes: 60
    if: github.repository_owner == 'apache' && github.event_name == 'schedule'
    steps:
      - name: Checkout code on master branch
        uses: actions/checkout@v2
        with:
          persist-credentials: false
          submodules: recursive
      - name: Tag commit
        run: |
          BRANCH_NAME=${GITHUB_REF##*/}
          echo "Tagging ${BRANCH_NAME}"
          git tag -f nightly-${BRANCH_NAME} HEAD
      - name: Push tags
        uses: ./.github/actions/github-push-action
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          tags: true
          force: true
          branch: master