feat: Reorganize install-deps in gsctl (#4154)

- [x] Split the gsctl install-deps commands for flex architecture - [x] Upgrade to the latest vineyard version - [x] Add global config file of vineyard version `V6D_VERSION` - [x] Use `V6D_VERSION` to build graphscope images - [ ] Use `V6D_VERSION` to build graphscope wheel packages - [x] Add `OEDestsSize` `IEDestsSize` `IOEDestsSize` interface in `ArrowProjectedFragment` - [x] CI related - [x] Install dependencies by `gsctl install-deps dev-analytical-java` in CI `gae.yml` - [x] rm `*dummy.yml` files from `.github/workflows`
alibaba · Sep 3, 2024 · 91690a5 · 91690a5
1 parent 73c1f6c
commit 91690a5
Show file tree

Hide file tree

Showing 50 changed files with 1,063 additions and 2,626 deletions.
diff --git a/.github/workflows/build-graphscope-dev-images.yml b/.github/workflows/build-graphscope-dev-images.yml
@@ -1,11 +1,10 @@
-name: Build GraphScope Development Images
+name: Build GraphScope Dev Images
 
 # build images for:
 # 1) graphscope-dev: including all dependencies for graphscope development env.
 # 2) vineyard-dev: including all vineyard-related dependencies that could compile graphscope analytical engine.
 # 3) vineyard-runtime: including all vineyard-related running dependencies.
-# Note that:
-#   Due to security considerations, we cannot use self-hosts runner(aarch64) when we configured the secret on github.
+# 4) `graphscope-dev:wheel`: including all dependencies for building graphscope wheel package
 on:
   workflow_dispatch:
     inputs:
@@ -23,6 +22,11 @@ on:
         required: true
         default: true
         type: boolean
+      build_graphscope_dev_wheel:
+        description: 'Whether to build graphscope-dev:wheel image'
+        required: true
+        default: true
+        type: boolean
   pull_request:
     branches:
       - main
@@ -38,8 +42,8 @@ env:
   REGISTRY: registry.cn-hongkong.aliyuncs.com
 
 jobs:
-  build-graphscope-dev-image-amd64:
-    if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_graphscope_dev == 'true') || (github.event_name == 'pull_request')
+  build-graphscope-dev-wheel-image-amd64:
+    if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_graphscope_dev_wheel == 'true') || (github.event_name == 'pull_request')
     runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v4
@@ -48,15 +52,15 @@ jobs:
 
     - name: Build Image
       run: |
-        # build graphscope dev image with specified v6d version
+        # build wheel image with specified v6d's version
         cd ${GITHUB_WORKSPACE}/k8s
         VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
         if [[ -n ${VINEYARD_VERSION} ]]; then
-          # graphscope/graphscope-dev:<v6d_version>-amd64
-          make graphscope-dev VINEYARD_VERSION=${VINEYARD_VERSION}
+          # graphscope/graphscope-dev:wheel-<v6d_version>-amd64
+          make dev-wheel VINEYARD_VERSION=${VINEYARD_VERSION}
         else
           # pull_request: use default vineyard_version
-          make graphscope-dev
+          make dev-wheel
         fi
 
     - name: Release Image
@@ -67,12 +71,12 @@ jobs:
       run: |
         echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
         VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
-        sudo docker tag graphscope/graphscope-dev:${VINEYARD_VERSION}-amd64 ${{ env.REGISTRY }}/graphscope/graphscope-dev:${VINEYARD_VERSION}-amd64
-        sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${VINEYARD_VERSION}-amd64
+        sudo docker tag graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-amd64 ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-amd64
+        sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:wheel-${VINEYARD_VERSION}-amd64
 
-  build-graphscope-dev-image-arm64:
-    runs-on: [self-hosted, Linux, ARM64]
-    if: false
+  build-graphscope-dev-image-amd64:
+    if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_graphscope_dev == 'true') || (github.event_name == 'pull_request')
+    runs-on: ubuntu-20.04
     steps:
     - uses: actions/checkout@v4
       with:
@@ -84,7 +88,7 @@ jobs:
         cd ${GITHUB_WORKSPACE}/k8s
         VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
         if [[ -n ${VINEYARD_VERSION} ]]; then
-          # graphscope/graphscope-dev:<v6d_version>-arm64
+          # graphscope/graphscope-dev:<v6d_version>-amd64
           make graphscope-dev VINEYARD_VERSION=${VINEYARD_VERSION}
         else
           # pull_request: use default vineyard_version
@@ -99,28 +103,8 @@ jobs:
       run: |
         echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
         VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
-        # graphscope-dev image
-        sudo docker tag graphscope/graphscope-dev:${VINEYARD_VERSION}-arm64 ${{ env.REGISTRY }}/graphscope/graphscope-dev:${VINEYARD_VERSION}-arm64
-        sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${VINEYARD_VERSION}-arm64
-
-  manifest-push-graphscope-dev-image:
-    runs-on: ubuntu-20.04
-    if: false
-    needs: [build-graphscope-dev-image-amd64, build-graphscope-dev-image-arm64]
-    steps:
-    - name: Create and Push Docker Manifest
-      env:
-        docker_password: ${{ secrets.DOCKER_PASSWORD }}
-        docker_username: ${{ secrets.DOCKER_USER }}
-      run: |
-        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
-        # manifest create
-        sudo docker manifest create \
-          ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }} \
-          ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }}-amd64 \
-          ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }}-arm64
-        # manifest push
-        sudo docker manifest push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${{ github.event.inputs.v6d_version }}
+        sudo docker tag graphscope/graphscope-dev:${VINEYARD_VERSION}-amd64 ${{ env.REGISTRY }}/graphscope/graphscope-dev:${VINEYARD_VERSION}-amd64
+        sudo docker push ${{ env.REGISTRY }}/graphscope/graphscope-dev:${VINEYARD_VERSION}-amd64
 
   build-vineyard-dev-image-amd64:
     if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_vineyard_dev == 'true') || (github.event_name == 'pull_request')
@@ -154,58 +138,6 @@ jobs:
         sudo docker tag graphscope/vineyard-dev:${VINEYARD_VERSION}-amd64 ${{ env.REGISTRY }}/graphscope/vineyard-dev:${VINEYARD_VERSION}-amd64
         sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${VINEYARD_VERSION}-amd64
 
-  build-vineyard-dev-image-arm64:
-    runs-on: [self-hosted, Linux, ARM64]
-    if: false
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-
-    - name: Build Image
-      run: |
-        # build vineyard dev image with specified v6d version
-        cd ${GITHUB_WORKSPACE}/k8s
-        VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
-        if [[ -n ${VINEYARD_VERSION} ]]; then
-          # graphscope/vineyard-dev:<version>-aarch64
-          make vineyard-dev VINEYARD_VERSION=${VINEYARD_VERSION}
-        else
-          # pull_request: use default vineyard_version
-          make vineyard-dev
-        fi
-
-    - name: Release Image
-      if: ${{ github.event_name == 'workflow_dispatch' }}
-      env:
-        docker_password: ${{ secrets.DOCKER_PASSWORD }}
-        docker_username: ${{ secrets.DOCKER_USER }}
-      run: |
-        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
-        VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
-        # vineyard-dev image
-        sudo docker tag graphscope/vineyard-dev:${VINEYARD_VERSION}-arm64 ${{ env.REGISTRY }}/graphscope/vineyard-dev:${VINEYARD_VERSION}-arm64
-        sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${VINEYARD_VERSION}-arm64
-
-  manifest-push-vineyard-dev-image:
-    runs-on: ubuntu-20.04
-    if: false
-    needs: [build-vineyard-dev-image-amd64, build-vineyard-dev-image-arm64]
-    steps:
-    - name: Create and Push Docker Manifest
-      env:
-        docker_password: ${{ secrets.DOCKER_PASSWORD }}
-        docker_username: ${{ secrets.DOCKER_USER }}
-      run: |
-        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
-        # manifest create
-        sudo docker manifest create \
-          ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }} \
-          ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }}-amd64 \
-          ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }}-arm64
-        # manifest push
-        sudo docker manifest push ${{ env.REGISTRY }}/graphscope/vineyard-dev:${{ github.event.inputs.v6d_version }}
-
   build-vineyard-runtime-image-amd64:
     # only trigger this step in 'workflow_dispatch' event, since the 'vineyard-dev' image isn't actually pushed in 'pull_request'
     if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }} || (github.event_name == 'pull_request')
@@ -239,74 +171,3 @@ jobs:
         VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
         sudo docker tag graphscope/vineyard-runtime:${VINEYARD_VERSION}-amd64 ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${VINEYARD_VERSION}-amd64
         sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${VINEYARD_VERSION}-amd64
-
-  build-vineyard-runtime-image-arm64:
-    runs-on: [self-hosted, Linux, ARM64]
-    # only trigger this step in 'workflow_dispatch' event,
-    # since the 'vineyard-dev' image isn't actually pushed in 'pull_request'
-    # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }} || (github.event_name == 'pull_request')
-    if: false
-
-    needs: [manifest-push-vineyard-dev-image]
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: true
-
-    - name: Build Image
-      run: |
-        # build vineyard dev image with specified v6d version
-        cd ${GITHUB_WORKSPACE}/k8s
-        VINEYARD_VERSION=${{ github.event.inputs.v6d_version }}
-        if [[ -n ${VINEYARD_VERSION} ]]; then
-          # graphscope/vineyard-runtime:<version>-aarch64
-          make vineyard-runtime VINEYARD_VERSION=${VINEYARD_VERSION}
-        else
-          # pull_request: use default vineyard_version
-          make vineyard-runtime
-        fi
-
-    - name: Release Image
-      env:
-        docker_password: ${{ secrets.DOCKER_PASSWORD }}
-        docker_username: ${{ secrets.DOCKER_USER }}
-      run: |
-        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
-        # aarch64
-        arch=$(uname -m)
-        # image tag
-        tag=${{ github.event.inputs.v6d_version }}-${arch}
-        # vineyard-runtime image
-        sudo docker tag graphscope/vineyard-runtime:${tag} ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag}
-        sudo docker push ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag}
-
-    - name: Clean Image
-      run: |
-        # aarch64
-        arch=$(uname -m)
-        # image tag
-        tag=${{ github.event.inputs.v6d_version }}-${arch}
-        # clean
-        sudo docker rmi -f graphscope/vineyard-runtime:${tag} || true
-        sudo docker rmi -f ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${tag} || true
-
-  manifest-push-vineyard-runtime-image:
-    runs-on: ubuntu-20.04
-    # if: ${{ github.event_name == 'workflow_dispatch' }} && ${{ github.event.inputs.build_vineyard_dev == 'true' }}
-    if: false
-
-    needs: [build-vineyard-runtime-image-amd64, build-vineyard-runtime-image-arm64]
-    steps:
-    - name: Create and Push Docker Manifest
-      env:
-        docker_password: ${{ secrets.DOCKER_PASSWORD }}
-        docker_username: ${{ secrets.DOCKER_USER }}
-      run: |
-        echo "${docker_password}" | sudo docker login --username="${docker_username}" ${{ env.REGISTRY }} --password-stdin
-        # manifest create
-        sudo docker manifest create \
-          ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${{ github.event.inputs.v6d_version }} \
-          ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${{ github.event.inputs.v6d_version }}-x86_64 \
-          ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${{ github.event.inputs.v6d_version }}-aarch64
-        # manifest push
-        sudo docker manifest push ${{ env.REGISTRY }}/graphscope/vineyard-runtime:${{ github.event.inputs.v6d_version }}
diff --git a/.github/workflows/build-graphscope-dev-wheel-images.yml b/.github/workflows/build-graphscope-dev-wheel-images.yml
diff --git a/.github/workflows/build-graphscope-wheels-macos.yml b/.github/workflows/build-graphscope-wheels-macos.yml
@@ -48,7 +48,7 @@ jobs:
 
         # Not sure when the first installation would fail, saying that the lock of python3.10 is taken
         # However the second trial would success.
-        python3 -m pip install click packaging
+        python3 -m pip install click
         python3 gsctl.py install-deps dev || true
         python3 gsctl.py install-deps dev
         echo "export PATH=/usr/local/opt/coreutils/libexec/gnubin:\$PATH" >> ~/.graphscope_env
@@ -166,7 +166,7 @@ jobs:
         sudo mkdir /opt/graphscope
         sudo chown -R $(id -u):$(id -g) /opt/graphscope
 
-        python3 -m pip install click packaging
+        python3 -m pip install click
         python3 gsctl.py install-deps dev || true
         python3 gsctl.py install-deps dev
 

diff --git a/.github/workflows/flex-interactive-dummy.yml b/.github/workflows/flex-interactive-dummy.yml
diff --git a/.github/workflows/gae-dummy.yml b/.github/workflows/gae-dummy.yml