build-graphscope-dev-images.yml

name: Build GraphScope Development and Wheel Images

# build images for:
# 1) wheel: including all dependencies for graphscope's wheel package.
# 2) graphscope-dev: including all dependencies for graphscope's development env.
# 3) vineyard-dev: including all vineyard-related dependencies that could compile graphscope analytical engine.
# 4) vineyard-runtime: including all vineyard-related running dependencies.
# Note that:
#   Due to security considerations, we cannot use self-hosts runner(aarch64) when we configured the secret on github.
on:
  workflow_dispatch:
    inputs:
      v6d_version:
        description: 'Version for Vineyard (v6d)'
        required: true
        default: 'main'
      build_wheel:
        description: 'Whether to build dev-wheel image'
        required: true
        default: true
        type: boolean
      build_graphscope_dev:
        description: 'Whether to build graphscope-dev image'
        required: true
        default: true
        type: boolean
      build_vineyard_dev:
        description: 'Whether to build vineyard-dev(runtime) image'
        required: true
        default: true
        type: boolean
  pull_request:
    branches:
      - main
    paths:
      - 'python/graphscope/gsctl/scripts/**'

concurrency:
  group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
  cancel-in-progress: true

env:
  REGISTRY: registry.cn-hongkong.aliyuncs.com

jobs:
  build-wheel-image-x86-64:
    runs-on: ubuntu-20.04
    if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_wheel == 'true') || (github.event_name == 'pull_request')

    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true

    - name: Build Image
      run: |
        # build wheel image with specified v6d's version
        cd ${GITHUB_WORKSPACE}/k8s
        VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
        if [[ -n ${VINEYARD_VERSION} ]]; then
          # graphscope/graphscope-dev:wheel-<version>-x86_64
          make dev-wheel VINEYARD_VERSION=${VINEYARD_VERSION}
        else
          # pull_request: use default vineyard_version
          make dev-wheel
        fi

    - name: Release Image
      if: ${{ github.event_name == 'workflow_dispatch' }}
      env:
        docker_password: ${{ secrets.DOCKER_PASSWORD }}
        docker_username: ${{ secrets.DOCKER_USER }}
      run: |
        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
        # x86_64
        arch=$(uname -m)
        # image tag
        tag=${{ github.event.inputs.v6d_version }}
        # dev-wheel image
        sudo docker tag graphscope/graphscope-dev:wheel-${tag}-${arch} ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag}
        sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag}

  build-wheel-image-aarch64:
    runs-on: [self-hosted, Linux, ARM64]
    # if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_wheel == 'true') || (github.event_name == 'pull_request')
    if: false

    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true

    - name: Build Image
      run: |
        # build wheel image with specified v6d's version
        cd ${GITHUB_WORKSPACE}/k8s
        VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
        if [[ -n ${VINEYARD_VERSION} ]]; then
          # graphscope/graphscope-dev:wheel-<version>-aarch64
          make dev-wheel VINEYARD_VERSION=${VINEYARD_VERSION}
        else
          # pull_request: use default vineyard_version
          make dev-wheel
        fi

    - name: Release Image
      if: ${{ github.event_name == 'workflow_dispatch' }}
      env:
        docker_password: ${{ secrets.DOCKER_PASSWORD }}
        docker_username: ${{ secrets.DOCKER_USER }}
      run: |
        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
        # aarch64
        arch=$(uname -m)
        # image tag
        tag=${{ github.event.inputs.v6d_version }}-${arch}
        # dev-wheel image
        sudo docker tag graphscope/graphscope-dev:wheel-${tag} ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag}
        sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag}

    - name: Clean Image
      run: |
        # aarch64
        arch=$(uname -m)
        # image tag
        tag=${{ github.event.inputs.v6d_version }}-${arch}
        # clean
        sudo docker rmi -f graphscope/graphscope-dev:wheel-${tag} || true
        sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${tag} || true

  manifest-push-wheel-image:
    runs-on: ubuntu-20.04
    # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_wheel == 'true' }}
    if: false

    needs: [build-wheel-image-x86-64, build-wheel-image-aarch64]
    steps:
    - name: Create and Push Docker Manifest
      env:
        docker_password: ${{ secrets.DOCKER_PASSWORD }}
        docker_username: ${{ secrets.DOCKER_USER }}
      run: |
        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
        # manifest create
        sudo docker manifest create \
          ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${{ github.event.inputs.v6d_version }} \
          ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${{ github.event.inputs.v6d_version }}-x86_64 \
          ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${{ github.event.inputs.v6d_version }}-aarch64
        # manifest push
        sudo docker manifest push ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${{ github.event.inputs.v6d_version }}

  build-graphscope-dev-image-x86-64:
    runs-on: ubuntu-20.04
    if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_graphscope_dev == 'true') || (github.event_name == 'pull_request')

    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true

    - name: Build Image
      run: |
        # build graphscope dev image with specified v6d's version
        cd ${GITHUB_WORKSPACE}/k8s
        VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
        if [[ -n ${VINEYARD_VERSION} ]]; then
          # graphscope/graphscope-dev:<version>-x86_64
          make graphscope-dev VINEYARD_VERSION=${VINEYARD_VERSION}
        else
          # pull_request: use default vineyard_version
          make graphscope-dev
        fi

    - name: Release Image
      if: ${{ github.event_name == 'workflow_dispatch' }}
      env:
        docker_password: ${{ secrets.DOCKER_PASSWORD }}
        docker_username: ${{ secrets.DOCKER_USER }}
      run: |
        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
        # x86_64
        arch=$(uname -m)
        # image tag
        tag=${{ github.event.inputs.v6d_version }}
        # graphscope-dev image
        sudo docker tag graphscope/graphscope-dev:${tag}-${arch} ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag}
        sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag}

  build-graphscope-dev-image-aarch64:
    runs-on: [self-hosted, Linux, ARM64]
    # if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_graphscope_dev == 'true') || (github.event_name == 'pull_request')
    if: false

    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true

    - name: Build Image
      run: |
        # build graphscope dev image with specified v6d's version
        cd ${GITHUB_WORKSPACE}/k8s
        VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
        if [[ -n ${VINEYARD_VERSION} ]]; then
          # graphscope/graphscope-dev:<version>-aarch64
          make graphscope-dev VINEYARD_VERSION=${VINEYARD_VERSION}
        else
          # pull_request: use default vineyard_version
          make graphscope-dev
        fi

    - name: Release Image
      if: ${{ github.event_name == 'workflow_dispatch' }}
      env:
        docker_password: ${{ secrets.DOCKER_PASSWORD }}
        docker_username: ${{ secrets.DOCKER_USER }}
      run: |
        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
        # aarch64
        arch=$(uname -m)
        # image tag
        tag=${{ github.event.inputs.v6d_version }}-${arch}
        # graphscope-dev image
        sudo docker tag graphscope/graphscope-dev:${tag} ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag}
        sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag}

    - name: Clean Image
      run: |
        # aarch64
        arch=$(uname -m)
        # image tag
        tag=${{ github.event.inputs.v6d_version }}-${arch}
        # clean
        sudo docker rmi -f graphscope/graphscope-dev:${tag} || true
        sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/graphscope-dev:${tag} || true

  manifest-push-graphscope-dev-image:
    runs-on: ubuntu-20.04
    # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_graphscope_dev == 'true' }}
    if: false

    needs: [build-graphscope-dev-image-x86-64, build-graphscope-dev-image-aarch64]
    steps:
    - name: Create and Push Docker Manifest
      env:
        docker_password: ${{ secrets.DOCKER_PASSWORD }}
        docker_username: ${{ secrets.DOCKER_USER }}
      run: |
        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
        # manifest create
        sudo docker manifest create \
          ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }} \
          ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }}-x86_64 \
          ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }}-aarch64
        # manifest push
        sudo docker manifest push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }}

  build-vineyard-dev-image-x86-64:
    runs-on: ubuntu-20.04
    if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_vineyard_dev == 'true') || (github.event_name == 'pull_request')

    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true

    - name: Build Image
      run: |
        # build vineyard dev image with specified v6d's version
        cd ${GITHUB_WORKSPACE}/k8s
        VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
        if [[ -n ${VINEYARD_VERSION} ]]; then
          # graphscope/vineyard-dev:<version>-x86_64
          make vineyard-dev VINEYARD_VERSION=${VINEYARD_VERSION}
        else
          # pull_request: use default vineyard_version
          make vineyard-dev
        fi

    - name: Release Image
      if: ${{ github.event_name == 'workflow_dispatch' }}
      env:
        docker_password: ${{ secrets.DOCKER_PASSWORD }}
        docker_username: ${{ secrets.DOCKER_USER }}
      run: |
        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
        # x86_64
        arch=$(uname -m)
        # image tag
        tag=${{ github.event.inputs.v6d_version }}
        # vineyard-dev image
        sudo docker tag graphscope/vineyard-dev:${tag}-${arch} ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag}
        sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag}

  build-vineyard-dev-image-aarch64:
    runs-on: [self-hosted, Linux, ARM64]
    # if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_vineyard_dev == 'true') || (github.event_name == 'pull_request')
    if: false

    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true

    - name: Build Image
      run: |
        # build vineyard dev image with specified v6d's version
        cd ${GITHUB_WORKSPACE}/k8s
        VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
        if [[ -n ${VINEYARD_VERSION} ]]; then
          # graphscope/vineyard-dev:<version>-aarch64
          make vineyard-dev VINEYARD_VERSION=${VINEYARD_VERSION}
        else
          # pull_request: use default vineyard_version
          make vineyard-dev
        fi

    - name: Release Image
      if: ${{ github.event_name == 'workflow_dispatch' }}
      env:
        docker_password: ${{ secrets.DOCKER_PASSWORD }}
        docker_username: ${{ secrets.DOCKER_USER }}
      run: |
        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
        # aarch64
        arch=$(uname -m)
        # image tag
        tag=${{ github.event.inputs.v6d_version }}-${arch}
        # vineyard-dev image
        sudo docker tag graphscope/vineyard-dev:${tag} ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag}
        sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag}

    - name: Clean Image
      run: |
        # aarch64
        arch=$(uname -m)
        # image tag
        tag=${{ github.event.inputs.v6d_version }}-${arch}
        # clean
        sudo docker rmi -f graphscope/vineyard-dev:${tag} || true
        sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/vineyard-dev:${tag} || true

  manifest-push-vineyard-dev-image:
    runs-on: ubuntu-20.04
    # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }}
    if: false

    needs: [build-vineyard-dev-image-x86-64, build-vineyard-dev-image-aarch64]
    steps:
    - name: Create and Push Docker Manifest
      env:
        docker_password: ${{ secrets.DOCKER_PASSWORD }}
        docker_username: ${{ secrets.DOCKER_USER }}
      run: |
        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
        # manifest create
        sudo docker manifest create \
          ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }} \
          ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }}-x86_64 \
          ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }}-aarch64
        # manifest push
        sudo docker manifest push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }}

  build-vineyard-runtime-image-x86-64:
    runs-on: ubuntu-20.04
    # only trigger this step in 'workflow_dispatch' event,
    # since the 'vineyard-dev' image isn't actually pushed in 'pull_request'
    if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }} || (github.event_name == 'pull_request')

    needs: [build-vineyard-dev-image-x86-64]
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true

    - name: Build Image
      run: |
        # build vineyard runtime image with specified v6d's version
        cd ${GITHUB_WORKSPACE}/k8s
        VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
        if [[ -n ${VINEYARD_VERSION} ]]; then
          # graphscope/vineyard-runtime:<version>-x86_64
          make vineyard-runtime VINEYARD_VERSION=${VINEYARD_VERSION}
        else
          # pull_request: use default vineyard_version
          make vineyard-runtime
        fi

    - name: Release Image
      if: ${{ github.event_name == 'workflow_dispatch' }}
      env:
        docker_password: ${{ secrets.DOCKER_PASSWORD }}
        docker_username: ${{ secrets.DOCKER_USER }}
      run: |
        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
        # x86_64
        arch=$(uname -m)
        # image tag
        tag=${{ github.event.inputs.v6d_version }}
        # vineyard-runtime image
        sudo docker tag graphscope/vineyard-runtime:${tag}-${arch} ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag}
        sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag}

  build-vineyard-runtime-image-aarch64:
    runs-on: [self-hosted, Linux, ARM64]
    # only trigger this step in 'workflow_dispatch' event,
    # since the 'vineyard-dev' image isn't actually pushed in 'pull_request'
    # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }} || (github.event_name == 'pull_request')
    if: false

    needs: [manifest-push-vineyard-dev-image]
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true

    - name: Build Image
      run: |
        # build vineyard dev image with specified v6d's version
        cd ${GITHUB_WORKSPACE}/k8s
        VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
        if [[ -n ${VINEYARD_VERSION} ]]; then
          # graphscope/vineyard-runtime:<version>-aarch64
          make vineyard-runtime VINEYARD_VERSION=${VINEYARD_VERSION}
        else
          # pull_request: use default vineyard_version
          make vineyard-runtime
        fi

    - name: Release Image
      env:
        docker_password: ${{ secrets.DOCKER_PASSWORD }}
        docker_username: ${{ secrets.DOCKER_USER }}
      run: |
        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
        # aarch64
        arch=$(uname -m)
        # image tag
        tag=${{ github.event.inputs.v6d_version }}-${arch}
        # vineyard-runtime image
        sudo docker tag graphscope/vineyard-runtime:${tag} ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag}
        sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag}

    - name: Clean Image
      run: |
        # aarch64
        arch=$(uname -m)
        # image tag
        tag=${{ github.event.inputs.v6d_version }}-${arch}
        # clean
        sudo docker rmi -f graphscope/vineyard-runtime:${tag} || true
        sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag} || true

  manifest-push-vineyard-runtime-image:
    runs-on: ubuntu-20.04
    # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }}
    if: false

    needs: [build-vineyard-runtime-image-x86-64, build-vineyard-runtime-image-aarch64]
    steps:
    - name: Create and Push Docker Manifest
      env:
        docker_password: ${{ secrets.DOCKER_PASSWORD }}
        docker_username: ${{ secrets.DOCKER_USER }}
      run: |
        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
        # manifest create
        sudo docker manifest create \
          ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${{ github.event.inputs.v6d_version }} \
          ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${{ github.event.inputs.v6d_version }}-x86_64 \
          ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${{ github.event.inputs.v6d_version }}-aarch64
        # manifest push
        sudo docker manifest push ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${{ github.event.inputs.v6d_version }}