diff --git a/.github/workflows/build-graphscope-dev-images.yml b/.github/workflows/build-graphscope-dev-images.yml index 569448ff33d1..c47810203b8a 100644 --- a/.github/workflows/build-graphscope-dev-images.yml +++ b/.github/workflows/build-graphscope-dev-images.yml @@ -48,7 +48,7 @@ jobs: if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_wheel == 'true') || (github.event_name == 'pull_request') steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true @@ -86,7 +86,7 @@ jobs: if: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true @@ -154,7 +154,7 @@ jobs: if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_graphscope_dev == 'true') || (github.event_name == 'pull_request') steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true @@ -192,7 +192,7 @@ jobs: if: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true @@ -260,7 +260,7 @@ jobs: if: (github.event_name == 'workflow_dispatch' && github.event.inputs.build_vineyard_dev == 'true') || (github.event_name == 'pull_request') steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true @@ -298,7 +298,7 @@ jobs: if: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true @@ -369,7 +369,7 @@ jobs: needs: [build-vineyard-dev-image-x86-64] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true @@ -410,7 +410,7 @@ jobs: needs: [manifest-push-vineyard-dev-image] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true diff --git a/.github/workflows/build-graphscope-images-linux.yml b/.github/workflows/build-graphscope-images-linux.yml index 94485bf7d19c..dd90bd185c70 100644 --- a/.github/workflows/build-graphscope-images-linux.yml +++ b/.github/workflows/build-graphscope-images-linux.yml @@ -35,7 +35,7 @@ jobs: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true @@ -172,7 +172,7 @@ jobs: # if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') || (github.pull_request == 'pull_request') || (github.event_name == 'workflow_dispatch') if: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true @@ -347,7 +347,7 @@ jobs: needs: [build-image-x86-64, build-image-aarch64] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true diff --git a/.github/workflows/build-graphscope-manylinux-ext-images.yml b/.github/workflows/build-graphscope-manylinux-ext-images.yml index 355a3371054a..0c2844ab23ae 100644 --- a/.github/workflows/build-graphscope-manylinux-ext-images.yml +++ b/.github/workflows/build-graphscope-manylinux-ext-images.yml @@ -21,7 +21,7 @@ jobs: build-manylinux-ext-image-x86-64: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true @@ -50,7 +50,7 @@ jobs: runs-on: [self-hosted, Linux, ARM64] if: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true diff --git a/.github/workflows/build-graphscope-wheels-linux.yml b/.github/workflows/build-graphscope-wheels-linux.yml index 2f13331bc8de..b56be6542a0a 100644 --- a/.github/workflows/build-graphscope-wheels-linux.yml +++ b/.github/workflows/build-graphscope-wheels-linux.yml @@ -26,7 +26,7 @@ jobs: runs-on: [self-hosted, Linux, ARM64] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true @@ -66,7 +66,7 @@ jobs: # mv ${GITHUB_WORKSPACE}/coordinator/dist/*.whl ${GITHUB_WORKSPACE}/upload_pypi/ - name: Upload Artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: wheel-arm64-${{ github.sha }} path: | @@ -98,7 +98,7 @@ jobs: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true @@ -146,7 +146,7 @@ jobs: mv ${GITHUB_WORKSPACE}/coordinator/dist/*.whl ${GITHUB_WORKSPACE}/upload_pypi/ - name: Upload Artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: wheel-${{ github.sha }} path: | @@ -179,11 +179,11 @@ jobs: needs: [build-wheels] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: path: artifacts @@ -250,7 +250,7 @@ jobs: python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: path: artifacts @@ -306,7 +306,7 @@ jobs: --shm-size 4096m steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: path: artifacts diff --git a/.github/workflows/build-graphscope-wheels-macos.yml b/.github/workflows/build-graphscope-wheels-macos.yml index c834207d56f7..bc7cb7362269 100644 --- a/.github/workflows/build-graphscope-wheels-macos.yml +++ b/.github/workflows/build-graphscope-wheels-macos.yml @@ -25,7 +25,7 @@ jobs: python-version: ['3.11'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true @@ -135,7 +135,7 @@ jobs: uses: mxschmitt/action-tmate@v2 - name: Upload Artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: wheel-macos-${{ github.sha }} path: graphscope.tar.gz @@ -149,7 +149,7 @@ jobs: python-version: ['3.8', '3.9', '3.10', '3.11'] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true @@ -221,7 +221,7 @@ jobs: tar -zcf client.tar.gz python/dist/ - name: Upload Artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: wheel-macos-${{ github.sha }}-${{ matrix.python-version }} path: client.tar.gz @@ -237,7 +237,7 @@ jobs: python-version: ['3.8', '3.9', '3.10', '3.11'] steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: path: artifacts @@ -285,7 +285,7 @@ jobs: python-version: ['3.8', '3.9', '3.10', '3.11'] steps: - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: path: artifacts diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index c125bdc4a1ee..605fd4cb2605 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -24,7 +24,7 @@ jobs: contents: write steps: - name: Checkout Code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: ${{ github.event.pull_request.head.repo.full_name }} ref: ${{ github.event.pull_request.head.ref }} diff --git a/.github/workflows/flex-interactive.yml b/.github/workflows/flex-interactive.yml index eb5dc514027e..18555bb44fb7 100644 --- a/.github/workflows/flex-interactive.yml +++ b/.github/workflows/flex-interactive.yml @@ -31,7 +31,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Add envs to GITHUB_ENV run: | diff --git a/.github/workflows/gae.yml b/.github/workflows/gae.yml index 8145b477dc73..da8ad6ee82a6 100644 --- a/.github/workflows/gae.yml +++ b/.github/workflows/gae.yml @@ -32,7 +32,7 @@ jobs: container: image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.22.0 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install latest vineyard if: false diff --git a/.github/workflows/gaia.yml b/.github/workflows/gaia.yml index 5b288af9e91e..918c377241c9 100644 --- a/.github/workflows/gaia.yml +++ b/.github/workflows/gaia.yml @@ -24,7 +24,7 @@ jobs: gaia-test: runs-on: [self-hosted, manylinux2014] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true @@ -113,7 +113,7 @@ jobs: - name: Upload GIE log if: failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 continue-on-error: true with: name: gie-log diff --git a/.github/workflows/gss.yml b/.github/workflows/gss.yml index 6421091dde1d..9652a1f792a5 100644 --- a/.github/workflows/gss.yml +++ b/.github/workflows/gss.yml @@ -44,7 +44,7 @@ jobs: runs-on: [self-hosted, manylinux2014] if: ${{ github.repository == 'alibaba/GraphScope' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Detect the tmate session run: | @@ -87,7 +87,7 @@ jobs: mvn test -P gremlin-test - name: Upload tools for helm test to Artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: groot path: | @@ -103,7 +103,7 @@ jobs: JAVA_HOME: /usr/lib/jvm/default-java GS_TEST_DIR: ${{ github.workspace }}/gstest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true @@ -114,7 +114,7 @@ jobs: exit 1 fi - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: groot path: artifacts @@ -231,7 +231,7 @@ jobs: - name: upload the k8s logs to artifact if: ${{ failure() }} - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 continue-on-error: true with: name: k8s-test-logs diff --git a/.github/workflows/interactive.yml b/.github/workflows/interactive.yml index 81491b240b75..3505505d0fb5 100644 --- a/.github/workflows/interactive.yml +++ b/.github/workflows/interactive.yml @@ -30,7 +30,7 @@ jobs: container: image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.23.0 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/cache@v3 with: @@ -372,7 +372,7 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install dependencies run: | @@ -400,7 +400,7 @@ jobs: BUILD_TEST: [ON, OFF] BUILD_ODPS_FRAGMENT_LOADER: [ON, OFF] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build run: | @@ -416,7 +416,7 @@ jobs: container: image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.23.0 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Download aocc compiler run: | @@ -445,7 +445,7 @@ jobs: container: image: registry.cn-hongkong.aliyuncs.com/graphscope/graphscope-dev:v0.23.0 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Install latest libgrape-lite if: false diff --git a/.github/workflows/k8s-ci-dummy.yml b/.github/workflows/k8s-ci-dummy.yml index f24c6e19351e..78857883c413 100644 --- a/.github/workflows/k8s-ci-dummy.yml +++ b/.github/workflows/k8s-ci-dummy.yml @@ -37,7 +37,7 @@ jobs: gie-function-test: ${{ steps.filter.outputs.gie-function-test }} steps: # For push it's necessary to checkout the code - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # For pull requests it's not necessary to checkout the code - uses: dorny/paths-filter@v2 id: filter diff --git a/.github/workflows/k8s-ci.yml b/.github/workflows/k8s-ci.yml index a2f02b552c90..b9805782fb00 100644 --- a/.github/workflows/k8s-ci.yml +++ b/.github/workflows/k8s-ci.yml @@ -52,7 +52,7 @@ jobs: gie-function-test: ${{ steps.filter.outputs.gie-function-test }} steps: # For push it's necessary to checkout the code - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Detect the tmate session run: | @@ -202,7 +202,7 @@ jobs: needs: [changes] if: ${{ github.repository == 'alibaba/GraphScope' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build Artifact run: | @@ -217,7 +217,7 @@ jobs: strip ${INSTALL_DIR}/builtin/*/*.so - name: Upload Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: analytical path: ${{ github.workspace }}/install @@ -229,7 +229,7 @@ jobs: #if: ${{ github.repository == 'alibaba/GraphScope' }} if: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/cache@v3 with: @@ -252,7 +252,7 @@ jobs: strip ${INSTALL_DIR}/builtin/*/*.so - name: Upload Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: analytical-java path: ${{ github.workspace }}/install @@ -263,7 +263,7 @@ jobs: needs: [changes] if: ${{ github.repository == 'alibaba/GraphScope' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/cache@v3 with: @@ -294,7 +294,7 @@ jobs: sccache --show-stats - name: Upload Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: interactive path: ${{ github.workspace }}/install @@ -305,7 +305,7 @@ jobs: needs: [changes] if: ${{ github.repository == 'alibaba/GraphScope' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true @@ -335,7 +335,7 @@ jobs: cp dist/*.whl ${{ github.workspace }}/install/ - name: Upload Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: learning path: ${{ github.workspace }}/install @@ -346,14 +346,14 @@ jobs: # needs: [changes] # if: false # Wheels of learning includes coordinator and client # steps: - # - uses: actions/checkout@v3g + # - uses: actions/checkout@v4g k8s-test: runs-on: [self-hosted, ubuntu2004] if: ${{ github.repository == 'alibaba/GraphScope' }} needs: [build-analytical, build-interactive, build-learning] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/cache@v3 with: @@ -362,7 +362,7 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: path: artifacts @@ -543,7 +543,7 @@ jobs: MODE: stop - name: Upload the k8s logs to artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ always() }} continue-on-error: true with: @@ -555,7 +555,7 @@ jobs: needs: [build-analytical, build-interactive, build-learning] if: ${{ needs.changes.outputs.gie-function-test == 'true' && github.repository == 'alibaba/GraphScope' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/cache@v3 with: @@ -564,7 +564,7 @@ jobs: restore-keys: | ${{ runner.os }}-pip- - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: path: artifacts @@ -619,7 +619,7 @@ jobs: runs-on: [self-hosted, manylinux2014] if: ${{ github.repository == 'alibaba/GraphScope' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/cache@v3 with: @@ -654,7 +654,7 @@ jobs: interactive_engine/executor/ir/target/release/${RPC_TARGET} - name: Upload Artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: gie-experimental path: | @@ -666,9 +666,9 @@ jobs: runs-on: [self-hosted, ubuntu2004] if: ${{ github.repository == 'alibaba/GraphScope' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: name: gie-experimental path: artifacts @@ -735,7 +735,7 @@ jobs: MODE: stop - name: upload the k8s logs to artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 continue-on-error: true if: ${{ always() }} with: diff --git a/.github/workflows/local-ci-dummy.yml b/.github/workflows/local-ci-dummy.yml index 9282d85fc6cc..0dc73235cc8c 100644 --- a/.github/workflows/local-ci-dummy.yml +++ b/.github/workflows/local-ci-dummy.yml @@ -37,7 +37,7 @@ jobs: gie-function-test: ${{ steps.filter.outputs.gie-function-test }} steps: # For push it's necessary to checkout the code - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # For pull requests it's not necessary to checkout the code - uses: dorny/paths-filter@v2 id: filter diff --git a/.github/workflows/local-ci.yml b/.github/workflows/local-ci.yml index 48f3530e8844..25a6b2f30021 100644 --- a/.github/workflows/local-ci.yml +++ b/.github/workflows/local-ci.yml @@ -52,7 +52,7 @@ jobs: gie-function-test: ${{ steps.filter.outputs.gie-function-test }} steps: # For push it's necessary to checkout the code - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Detect the tmate session run: | @@ -207,7 +207,7 @@ jobs: needs: [changes] if: ${{ github.repository == 'alibaba/GraphScope' }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true @@ -257,7 +257,7 @@ jobs: tar -zcf graphscope.tar.gz coordinator/dist/ - name: Upload Artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: wheel-${{ github.sha }} path: | @@ -270,9 +270,9 @@ jobs: if: ${{ github.repository == 'alibaba/GraphScope' }} needs: [build-wheels] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 with: path: artifacts @@ -320,7 +320,7 @@ jobs: - name: Upload GIE log if: failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 continue-on-error: true with: name: gie-log @@ -338,10 +338,10 @@ jobs: deployment: ["standalone", "distributed"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 if: ${{ needs.changes.outputs.gae-python == 'true' }} - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 if: ${{ needs.changes.outputs.gae-python == 'true' }} with: path: artifacts @@ -407,7 +407,7 @@ jobs: matrix: deployment: ["standalone", "distributed"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 if: ${{ needs.changes.outputs.networkx == 'true' }} - uses: dorny/paths-filter@v2 @@ -429,7 +429,7 @@ jobs: - 'python/graphscope/nx/convert_matrix.py' - 'python/graphscope/nx/tests/convert/**' - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 if: ${{ needs.changes.outputs.networkx == 'true' }} with: path: artifacts @@ -496,7 +496,7 @@ jobs: matrix: deployment: ["standalone", "distributed"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 if: ${{ needs.changes.outputs.networkx == 'true' }} - uses: dorny/paths-filter@v2 @@ -516,7 +516,7 @@ jobs: io: - 'python/graphscope/nx/readwrite/**' - - uses: actions/download-artifact@v3 + - uses: actions/download-artifact@v4 if: ${{ needs.changes.outputs.networkx == 'true' }} with: path: artifacts diff --git a/.github/workflows/networkx-forward-algo-nightly.yml b/.github/workflows/networkx-forward-algo-nightly.yml index d745472b43cf..f395b0a4a7a3 100644 --- a/.github/workflows/networkx-forward-algo-nightly.yml +++ b/.github/workflows/networkx-forward-algo-nightly.yml @@ -22,7 +22,7 @@ jobs: --shm-size 4096m steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build GAE and coordinator run: | diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index ecd8d8333559..1d2fa34f47c6 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -20,7 +20,7 @@ jobs: matrix: os: [ubuntu-22.04, macos-latest] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true diff --git a/.github/workflows/pegasus.yml b/.github/workflows/pegasus.yml index ff4f204c2c9c..4d96fb74b3c8 100644 --- a/.github/workflows/pegasus.yml +++ b/.github/workflows/pegasus.yml @@ -23,7 +23,7 @@ jobs: pegasus-build: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build & Test run: | @@ -51,7 +51,7 @@ jobs: # k8s-test: # runs-on: [self-hosted, ubuntu2004] # steps: -# - uses: actions/checkout@v3 +# - uses: actions/checkout@v4 # # - name: Detect the tmate session # run: | diff --git a/.github/workflows/pr-check.yml b/.github/workflows/pr-check.yml index 8913ced96921..fa6baad55298 100644 --- a/.github/workflows/pr-check.yml +++ b/.github/workflows/pr-check.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Checkout Code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: repository: ${{ github.event.pull_request.head.repo.full_name }} ref: ${{ github.event.pull_request.head.ref }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 69851500143b..77adb6251a79 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,7 +27,7 @@ jobs: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Build Wheel Package run: | @@ -49,7 +49,7 @@ jobs: tar -zcf gsctl.tar.gz ${GITHUB_WORKSPACE}/upload_pypi/*.whl - name: Upload Artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: gsctl-${{ github.sha }} path: | @@ -69,7 +69,7 @@ jobs: runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Add envs to GITHUB_ENV run: | @@ -102,7 +102,7 @@ jobs: if: (github.ref == 'refs/heads/main' && github.repository == 'alibaba/GraphScope') || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope') runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Add envs to GITHUB_ENV run: | @@ -155,7 +155,7 @@ jobs: if: false runs-on: [self-hosted, Linux, ARM64] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Add envs to GITHUB_ENV run: | @@ -232,7 +232,7 @@ jobs: needs: [build-gss-image-x86-64, build-gss-image-aarch64] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true @@ -261,7 +261,7 @@ jobs: if: ${{ github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v') && github.repository == 'alibaba/GraphScope' }} runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 # this step is needed for update index.yaml - name: Download Released Charts diff --git a/CITATION.cff b/CITATION.cff index c4fab3ed774e..bb6f822f32f8 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -2,31 +2,43 @@ cff-version: 1.2.0 message: >- If you use this software, please cite our paper using the metadata from this file. -title: 'GraphScope: a unified engine for big graph processing' +title: 'GraphScope Flex: LEGO-like Graph Computing Stack' authors: - - given-names: Wenfei - family-names: Fan - affiliation: University of Edinburgh - given-names: Tao family-names: He affiliation: Alibaba Group + - given-names: Shuxian + family-names: Hu + affiliation: Alibaba Group - given-names: Longbin family-names: Lai affiliation: Alibaba Group - - given-names: Xue + - given-names: Dongze family-names: Li affiliation: Alibaba Group - - given-names: Yong + - given-names: Neng family-names: Li affiliation: Alibaba Group - - given-names: Zhao + - given-names: Xue family-names: Li affiliation: Alibaba Group - - given-names: Zhengqian - family-names: Qian + - given-names: Lexiao + family-names: Liu + affiliation: Alibaba Group + - given-names: Xiaojian + family-names: Luo affiliation: Alibaba Group - - given-names: Chao - family-names: Tian + - given-names: Bingqing + family-names: Lyu + affiliation: Alibaba Group + - given-names: Ke + family-names: Meng + affiliation: Alibaba Group + - given-names: Sijie + family-names: Shen + affiliation: Alibaba Group + - given-names: Su + family-names: Li affiliation: Alibaba Group - given-names: Lei family-names: Wang @@ -34,108 +46,106 @@ authors: - given-names: Jingbo family-names: Xu affiliation: Alibaba Group - - given-names: Qiang - family-names: Yin - affiliation: Alibaba Group - given-names: Wenyuan family-names: Yu affiliation: Alibaba Group + - given-names: Weibin + family-names: Zeng + affiliation: Alibaba Group + - given-names: Lei + family-names: Zhang + affiliation: Alibaba Group + - given-names: Siyuan + family-names: Zhang + affiliation: Alibaba Group - given-names: Jingren family-names: Zhou affiliation: Alibaba Group - - given-names: Diwen - family-names: Zhu + - given-names: Xiaoli + family-names: Zhou affiliation: Alibaba Group - - given-names: Rong + - given-names: Diwen family-names: Zhu affiliation: Alibaba Group license: Apache-2.0 identifiers: - type: doi - value: 10.14778/3476311.3476369 + value: 10.1145/3626246.3653383 repository-code: 'https://github.com/alibaba/GraphScope' url: 'https://graphscope.io' -abstract: >- - GraphScope is a system and a set of language extensions - that enable a new programming interface for large-scale - distributed graph computing. It generalizes previous graph - processing frameworks (e.g., Pregel, GraphX) and - distributed graph databases (e.g., JanusGraph, Neptune) in - two important ways: by exposing a unified programming - interface to a wide variety of graph computations such as - graph traversal, pattern matching, iterative algorithms, - and graph neural networks within a high-level programming - language; and by supporting the seamless integration of a - highly optimized graph engine in a general purpose - data-parallel computing system. - - A GraphScope program is a sequential program composed of - declarative data-parallel operators and can be written - using standard Python development tools. The system - automatically handles the parallelization and distributed - execution of programs on a cluster of machines. It - outperforms current state-of-the-art systems by enabling a - separate optimization (or family of optimizations) for - each graph operation in one carefully designed coherent - framework. We describe the design and implementation of - GraphScope and evaluate system performance using several - real-world applications. preferred-citation: type: article - title: 'GraphScope: a unified engine for big graph processing' + title: 'GraphScope Flex: LEGO-like Graph Computing Stack' authors: - - given-names: Wenfei - family-names: Fan - affiliation: University of Edinburgh - - given-names: Tao - family-names: He - affiliation: Alibaba Group - - given-names: Longbin - family-names: Lai - affiliation: Alibaba Group - - given-names: Xue - family-names: Li - affiliation: Alibaba Group - - given-names: Yong - family-names: Li - affiliation: Alibaba Group - - given-names: Zhao - family-names: Li - affiliation: Alibaba Group - - given-names: Zhengqian - family-names: Qian - affiliation: Alibaba Group - - given-names: Chao - family-names: Tian - affiliation: Alibaba Group - - given-names: Lei - family-names: Wang - affiliation: Alibaba Group - - given-names: Jingbo - family-names: Xu - affiliation: Alibaba Group - - given-names: Qiang - family-names: Yin - affiliation: Alibaba Group - - given-names: Wenyuan - family-names: Yu - affiliation: Alibaba Group - - given-names: Jingren - family-names: Zhou - affiliation: Alibaba Group - - given-names: Diwen - family-names: Zhu - affiliation: Alibaba Group - - given-names: Rong - family-names: Zhu - affiliation: Alibaba Group - year: 2021 - journal: "Proceedings of the VLDB Endowment" - doi: 10.14778/3476311.3476369 - month: 08 - start: 2879 - end: 2892 - volume: 14 - number: 12 + - given-names: Tao + family-names: He + affiliation: Alibaba Group + - given-names: Shuxian + family-names: Hu + affiliation: Alibaba Group + - given-names: Longbin + family-names: Lai + affiliation: Alibaba Group + - given-names: Dongze + family-names: Li + affiliation: Alibaba Group + - given-names: Neng + family-names: Li + affiliation: Alibaba Group + - given-names: Xue + family-names: Li + affiliation: Alibaba Group + - given-names: Lexiao + family-names: Liu + affiliation: Alibaba Group + - given-names: Xiaojian + family-names: Luo + affiliation: Alibaba Group + - given-names: Bingqing + family-names: Lyu + affiliation: Alibaba Group + - given-names: Ke + family-names: Meng + affiliation: Alibaba Group + - given-names: Sijie + family-names: Shen + affiliation: Alibaba Group + - given-names: Su + family-names: Li + affiliation: Alibaba Group + - given-names: Lei + family-names: Wang + affiliation: Alibaba Group + - given-names: Jingbo + family-names: Xu + affiliation: Alibaba Group + - given-names: Wenyuan + family-names: Yu + affiliation: Alibaba Group + - given-names: Weibin + family-names: Zeng + affiliation: Alibaba Group + - given-names: Lei + family-names: Zhang + affiliation: Alibaba Group + - given-names: Siyuan + family-names: Zhang + affiliation: Alibaba Group + - given-names: Jingren + family-names: Zhou + affiliation: Alibaba Group + - given-names: Xiaoli + family-names: Zhou + affiliation: Alibaba Group + - given-names: Diwen + family-names: Zhu + affiliation: Alibaba Group + year: 2024 + journal: "Companion of the 2024 International Conference on Management of Data" + doi: 10.1145/3626246.3653383 + start: 386 + end: 399 + number: 14 + issue: SIGMOD/PODS '24 publisher: - name: VLDB Endowment + name: "Association for Computing Machinery" diff --git a/README.md b/README.md index 61066d45d779..8808f56ed877 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,7 @@ GraphScope is a unified distributed graph computing platform that provides a one Visit our website at [graphscope.io](https://graphscope.io) to learn more. ## Latest News +- [31/07/2024] ๐ŸŽข Weโ€™ve launched a webpage visualizing GraphScopeโ€™s journey in graph computing. [Check it out!](https://graphscope.io/journey) - [30/05/2024] ๐Ÿ… GraphScope Flex set new record-breaking [SNB Interactive audit results](https://ldbcouncil.org/benchmarks/snb-interactive/), as announced by LDBC on [X (Twitter)](https://twitter.com/LDBCouncil/status/1795886732950294630)! - [25/03/2024] ๐Ÿ™Œ๐Ÿป We donated the graph file format [GraphAr](https://graphar.apache.org/) to [Apache Software Foundation](https://www.apache.org/) as an Incubating Project. - [05/02/2024] ๐ŸŽ‰ GraphScope Flex [paper](https://arxiv.org/abs/2312.12107) was accepted by [SIGMOD 2024](https://2024.sigmod.org/) Industry Track. See you in ๐Ÿ‡จ๐Ÿ‡ฑ! @@ -413,12 +414,14 @@ GraphScope is released under [Apache License 2.0](https://www.apache.org/license ## Publications +- Tao He, Shuxian Hu, Longbin Lai, Dongze Li, Neng Li, Xue Li, Lexiao Liu, Xiaojian Luo, Binqing Lyu, Ke Meng, Sijie Shen, Li Su, Lei Wang, Jingbo Xu, Wenyuan Yu, Weibin Zeng, Lei Zhang, Siyuan Zhang, Jingren Zhou, Xiaoli Zhou, Diwen Zhu. [GraphScope Flex: LEGO-like Graph Computing Stack](https://dl.acm.org/doi/10.1145/3626246.3653383). Companion of the 2024 International Conference on Management of Data (SIGMOD), industry, 2024. - Wenfei Fan, Tao He, Longbin Lai, Xue Li, Yong Li, Zhao Li, Zhengping Qian, Chao Tian, Lei Wang, Jingbo Xu, Youyang Yao, Qiang Yin, Wenyuan Yu, Jingren Zhou, Diwen Zhu, Rong Zhu. [GraphScope: A Unified Engine For Big Graph Processing](http://vldb.org/pvldb/vol14/p2879-qian.pdf). The 47th International Conference on Very Large Data Bases (VLDB), industry, 2021. - Jingbo Xu, Zhanning Bai, Wenfei Fan, Longbin Lai, Xue Li, Zhao Li, Zhengping Qian, Lei Wang, Yanyan Wang, Wenyuan Yu, Jingren Zhou. [GraphScope: A One-Stop Large Graph Processing System](http://vldb.org/pvldb/vol14/p2703-xu.pdf). The 47th International Conference on Very Large Data Bases (VLDB), demo, 2021 If you use this software, please cite our paper using the following metadata: ```bibtex +# GraphScope paper @article{fan2021graphscope, title={GraphScope: a unified engine for big graph processing}, author={Fan, Wenfei and He, Tao and Lai, Longbin and Li, Xue and Li, Yong and Li, Zhao and Qian, Zhengping and Tian, Chao and Wang, Lei and Xu, Jingbo and others}, @@ -429,6 +432,19 @@ If you use this software, please cite our paper using the following metadata: year={2021}, publisher={VLDB Endowment} } +# or GraphScope Flex paper +@inproceedings{10.1145/3626246.3653383, + title = {GraphScope Flex: LEGO-like Graph Computing Stack}, + author = {He, Tao and Hu, Shuxian and Lai, Longbin and Li, Dongze and Li, Neng and Li, Xue and Liu, Lexiao and Luo, Xiaojian and Lyu, Bingqing and Meng, Ke and Shen, Sijie and Su, Li and Wang, Lei and Xu, Jingbo and Yu, Wenyuan and Zeng, Weibin and Zhang, Lei and Zhang, Siyuan and Zhou, Jingren and Zhou, Xiaoli and Zhu, Diwen}, + year = {2024}, + isbn = {9798400704222}, + publisher = {Association for Computing Machinery}, + booktitle = {Companion of the 2024 International Conference on Management of Data}, + pages = {386โ€“399}, + numpages = {14}, + location = {Santiago AA, Chile}, + series = {SIGMOD/PODS '24} +} ``` ## Contributing diff --git a/analytical_engine/CMakeLists.txt b/analytical_engine/CMakeLists.txt index fa5853dce7d4..bac1a4b1492d 100644 --- a/analytical_engine/CMakeLists.txt +++ b/analytical_engine/CMakeLists.txt @@ -289,7 +289,7 @@ endif() execute_process(COMMAND python3 ../proto/proto_generator.py "${PROJECT_SOURCE_DIR}/proto" --cpp WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) -file(GLOB PROTO_CPP_FILES "proto/*.cc") +file(GLOB PROTO_CPP_FILES "proto/*.cc" "proto/error/*.cc") file(GLOB CORE_RPC_SRC_FILES "core/server/*.cc") # Add targets diff --git a/analytical_engine/apps/sssp/sssp_average_length.h b/analytical_engine/apps/sssp/sssp_average_length.h index e064bc5f576b..b1de0b5dba43 100644 --- a/analytical_engine/apps/sssp/sssp_average_length.h +++ b/analytical_engine/apps/sssp/sssp_average_length.h @@ -109,7 +109,6 @@ class SSSPAverageLength bool update_sum = false; pair_msg_t msg; - int msg_cnt = 0; while (messages.GetMessage(msg)) { bool is_vertex_msg = msg.first; if (is_vertex_msg) { @@ -129,7 +128,6 @@ class SSSPAverageLength fid_t fid = (fid_t)(std::get<0>(msg.second)); ctx.all_sums[fid] = std::get<2>(msg.second); } - msg_cnt++; } for (auto& it : updated_map) { diff --git a/analytical_engine/core/error.h b/analytical_engine/core/error.h index a5a1e0c6f60f..6d7f3f086bf4 100644 --- a/analytical_engine/core/error.h +++ b/analytical_engine/core/error.h @@ -24,7 +24,7 @@ #include "vineyard/graph/utils/error.h" // IWYU pragma: export -#include "proto/error_codes.pb.h" // IWYU pragma: export +#include "proto/error/coordinator.pb.h" // IWYU pragma: export namespace gs { @@ -35,21 +35,28 @@ inline rpc::Code ErrorCodeToProto(vineyard::ErrorCode ec) { case vineyard::ErrorCode::kVineyardError: return rpc::Code::VINEYARD_ERROR; case vineyard::ErrorCode::kNetworkError: - return rpc::Code::NETWORK_ERROR; + return rpc::Code::NETWORK_FAILURE; case vineyard::ErrorCode::kUnimplementedMethod: - return rpc::Code::UNIMPLEMENTED_ERROR; + return rpc::Code::UNIMPLEMENTED; default: return rpc::Code::ANALYTICAL_ENGINE_INTERNAL_ERROR; } } +inline std::string formatEnumValue(const vineyard::ErrorCode& value) { + std::stringstream ss; + // Format the integer value with leading zeros (4 digits) + ss << "02-" << std::setw(4) << std::setfill('0') << static_cast(value); + return ss.str(); +} + #ifndef __FRAME_MAKE_GS_ERROR #define __FRAME_MAKE_GS_ERROR(var, code, msg) \ do { \ std::stringstream TOKENPASTE2(_ss, __LINE__); \ vineyard::backtrace_info::backtrace(TOKENPASTE2(_ss, __LINE__), true); \ LOG(ERROR) << "graphscope error in frame: code = " \ - << static_cast(code) << " at " \ + << gs::formatEnumValue(code) << " at " \ << (std::string(__FILE__) + ":" + std::to_string(__LINE__) + \ ": " + std::string(__FUNCTION__)) \ << " -> " << (msg) \ @@ -68,7 +75,7 @@ inline rpc::Code ErrorCodeToProto(vineyard::ErrorCode ec) { std::stringstream TOKENPASTE2(_ss, __LINE__); \ vineyard::backtrace_info::backtrace(TOKENPASTE2(_ss, __LINE__), true); \ LOG(ERROR) << "graphscope error in frame: code = " \ - << static_cast(code) << " at " \ + << gs::formatEnumValue(code) << " at " \ << (std::string(__FILE__) + ":" + std::to_string(__LINE__) + \ ": " + std::string(__FUNCTION__)) \ << " -> " << (msg) \ diff --git a/analytical_engine/core/server/dispatcher.h b/analytical_engine/core/server/dispatcher.h index ecbffda33805..885333238d7a 100644 --- a/analytical_engine/core/server/dispatcher.h +++ b/analytical_engine/core/server/dispatcher.h @@ -25,7 +25,7 @@ #include "grape/worker/comm_spec.h" #include "vineyard/common/util/blocking_queue.h" -#include "proto/error_codes.pb.h" +#include "proto/error/coordinator.pb.h" #include "proto/graph_def.pb.h" #include "proto/types.pb.h" diff --git a/analytical_engine/core/server/graphscope_service.cc b/analytical_engine/core/server/graphscope_service.cc index a29f32ddad89..3fe47bf60263 100644 --- a/analytical_engine/core/server/graphscope_service.cc +++ b/analytical_engine/core/server/graphscope_service.cc @@ -27,7 +27,7 @@ #include "core/server/rpc_utils.h" #include "proto/attr_value.pb.h" -#include "proto/error_codes.pb.h" +#include "proto/error/coordinator.pb.h" #include "proto/graph_def.pb.h" #include "proto/message.pb.h" #include "proto/op_def.pb.h" diff --git a/analytical_engine/core/worker/default_property_worker.h b/analytical_engine/core/worker/default_property_worker.h index 28f89b9ffcc1..5fd89a601507 100644 --- a/analytical_engine/core/worker/default_property_worker.h +++ b/analytical_engine/core/worker/default_property_worker.h @@ -95,8 +95,6 @@ class DefaultPropertyWorker { context_->Init(messages_, std::forward(args)...); - int round = 0; - messages_.Start(); messages_.StartARound(); @@ -114,7 +112,6 @@ class DefaultPropertyWorker { while (!messages_.ToTerminate()) { t = grape::GetCurrentTime(); - round++; messages_.StartARound(); app_->IncEval(graph, *context_, messages_); diff --git a/analytical_engine/core/worker/default_worker.h b/analytical_engine/core/worker/default_worker.h index 9c4587951074..7e304e56f5aa 100644 --- a/analytical_engine/core/worker/default_worker.h +++ b/analytical_engine/core/worker/default_worker.h @@ -95,8 +95,6 @@ class DefaultWorker { context_->Init(messages_, std::forward(args)...); - int round = 0; - messages_.Start(); messages_.StartARound(); @@ -114,7 +112,6 @@ class DefaultWorker { while (!messages_.ToTerminate()) { t = grape::GetCurrentTime(); - round++; messages_.StartARound(); app_->IncEval(graph, *context_, messages_); diff --git a/analytical_engine/core/worker/parallel_property_worker.h b/analytical_engine/core/worker/parallel_property_worker.h index 3455732a24ea..54f3b81ca822 100644 --- a/analytical_engine/core/worker/parallel_property_worker.h +++ b/analytical_engine/core/worker/parallel_property_worker.h @@ -95,8 +95,6 @@ class ParallelPropertyWorker { context_->Init(messages_, std::forward(args)...); - int round = 0; - messages_.Start(); messages_.StartARound(); @@ -114,7 +112,6 @@ class ParallelPropertyWorker { while (!messages_.ToTerminate()) { t = grape::GetCurrentTime(); - round++; messages_.StartARound(); app_->IncEval(*graph_, *context_, messages_); diff --git a/analytical_engine/core/worker/property_auto_worker.h b/analytical_engine/core/worker/property_auto_worker.h index 11e71c948da0..07b5ce0b19d4 100644 --- a/analytical_engine/core/worker/property_auto_worker.h +++ b/analytical_engine/core/worker/property_auto_worker.h @@ -90,8 +90,6 @@ class PropertyAutoWorker { context_->Init(messages_, std::forward(args)...); - int round = 0; - messages_.Start(); messages_.StartARound(); @@ -109,7 +107,6 @@ class PropertyAutoWorker { while (!messages_.ToTerminate()) { t = grape::GetCurrentTime(); - round++; messages_.StartARound(); app_->IncEval(graph, *context_); diff --git a/charts/graphscope-store/templates/coordinator/statefulset.yaml b/charts/graphscope-store/templates/coordinator/statefulset.yaml index e8b0f999e475..8a088ba67a97 100644 --- a/charts/graphscope-store/templates/coordinator/statefulset.yaml +++ b/charts/graphscope-store/templates/coordinator/statefulset.yaml @@ -89,6 +89,14 @@ spec: env: - name: ROLE value: "coordinator" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - name: inner-rpc containerPort: 55555 diff --git a/charts/graphscope-store/templates/frontend/statefulset.yaml b/charts/graphscope-store/templates/frontend/statefulset.yaml index 6bd079d117ff..4689313603c8 100644 --- a/charts/graphscope-store/templates/frontend/statefulset.yaml +++ b/charts/graphscope-store/templates/frontend/statefulset.yaml @@ -89,6 +89,14 @@ spec: env: - name: ROLE value: "frontend" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - name: grpc containerPort: {{ .Values.frontend.service.servicePort }} diff --git a/charts/graphscope-store/templates/onepod/statefulset.yaml b/charts/graphscope-store/templates/onepod/statefulset.yaml index 0143756f872b..d05c928a0a46 100644 --- a/charts/graphscope-store/templates/onepod/statefulset.yaml +++ b/charts/graphscope-store/templates/onepod/statefulset.yaml @@ -89,6 +89,14 @@ spec: env: - name: ROLE value: "" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - name: gremlin containerPort: {{ .Values.frontend.service.gremlinPort }} diff --git a/charts/graphscope-store/templates/store/statefulset-backup.yaml b/charts/graphscope-store/templates/store/statefulset-backup.yaml index 0f6bf76f3112..aedefc5b5bb1 100644 --- a/charts/graphscope-store/templates/store/statefulset-backup.yaml +++ b/charts/graphscope-store/templates/store/statefulset-backup.yaml @@ -93,6 +93,14 @@ spec: env: - name: ROLE value: "store" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - name: inner-rpc containerPort: 55555 diff --git a/charts/graphscope-store/templates/store/statefulset.yaml b/charts/graphscope-store/templates/store/statefulset.yaml index f75a0c5a8141..9bb943aa27c9 100644 --- a/charts/graphscope-store/templates/store/statefulset.yaml +++ b/charts/graphscope-store/templates/store/statefulset.yaml @@ -89,6 +89,14 @@ spec: env: - name: ROLE value: "store" + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name ports: - name: inner-rpc containerPort: 55555 diff --git a/coordinator/gscoordinator/kubernetes_launcher.py b/coordinator/gscoordinator/kubernetes_launcher.py index aca70b6475a9..978b1168068f 100644 --- a/coordinator/gscoordinator/kubernetes_launcher.py +++ b/coordinator/gscoordinator/kubernetes_launcher.py @@ -1519,9 +1519,11 @@ def create_learning_instance(self, object_id, handle, config, learning_backend): pod_name_list, pod_host_ip_list, object_id, handle, config ) elif learning_backend == message_pb2.LearningBackend.GRAPHLEARN_TORCH: - pod_name_list, pod_ip_list, pod_host_ip_list = ( - self._allocate_graphlearn_torch_engine(object_id) - ) + ( + pod_name_list, + pod_ip_list, + pod_host_ip_list, + ) = self._allocate_graphlearn_torch_engine(object_id) if not pod_name_list or not pod_host_ip_list: raise RuntimeError("Failed to allocate learning engine") return self._distribute_graphlearn_torch_process( diff --git a/coordinator/gscoordinator/op_executor.py b/coordinator/gscoordinator/op_executor.py index 8e9f33f27492..a1d179c337ac 100644 --- a/coordinator/gscoordinator/op_executor.py +++ b/coordinator/gscoordinator/op_executor.py @@ -29,7 +29,7 @@ from graphscope.proto import message_pb2 from graphscope.proto import op_def_pb2 from graphscope.proto import types_pb2 -from graphscope.proto.error_codes_pb2 import OK +from graphscope.proto.error.coordinator_pb2 import OK from gscoordinator.launcher import AbstractLauncher from gscoordinator.monitor import Monitor diff --git a/coordinator/gscoordinator/servicer/graphscope_one/service.py b/coordinator/gscoordinator/servicer/graphscope_one/service.py index 904f27d13c54..6adca8df7f91 100644 --- a/coordinator/gscoordinator/servicer/graphscope_one/service.py +++ b/coordinator/gscoordinator/servicer/graphscope_one/service.py @@ -47,9 +47,9 @@ from graphscope.framework.utils import i_to_attr from graphscope.framework.utils import s_to_attr from graphscope.proto import coordinator_service_pb2_grpc -from graphscope.proto import error_codes_pb2 from graphscope.proto import message_pb2 from graphscope.proto import types_pb2 +from graphscope.proto.error import coordinator_pb2 from gscoordinator.dag_manager import DAGManager from gscoordinator.dag_manager import GSEngine @@ -235,29 +235,29 @@ def _RunStep(self, request_iterator, context): while not dag_manager.empty(): run_dag_on, dag, dag_bodies = dag_manager.next_dag() - error_code = error_codes_pb2.COORDINATOR_INTERNAL_ERROR + error_code = coordinator_pb2.COORDINATOR_INTERNAL_ERROR head, bodies = None, None try: # run on analytical engine if run_dag_on == GSEngine.analytical_engine: # need dag_bodies to load graph from pandas/numpy - error_code = error_codes_pb2.ANALYTICAL_ENGINE_INTERNAL_ERROR + error_code = coordinator_pb2.ANALYTICAL_ENGINE_INTERNAL_ERROR head, bodies = self._operation_executor.run_on_analytical_engine( dag, dag_bodies, loader_op_bodies ) # run on interactive engine elif run_dag_on == GSEngine.interactive_engine: - error_code = error_codes_pb2.INTERACTIVE_ENGINE_INTERNAL_ERROR + error_code = coordinator_pb2.INTERACTIVE_ENGINE_INTERNAL_ERROR head, bodies = self._operation_executor.run_on_interactive_engine( dag ) # run on learning engine elif run_dag_on == GSEngine.learning_engine: - error_code = error_codes_pb2.LEARNING_ENGINE_INTERNAL_ERROR + error_code = coordinator_pb2.LEARNING_ENGINE_INTERNAL_ERROR head, bodies = self._operation_executor.run_on_learning_engine(dag) # run on coordinator elif run_dag_on == GSEngine.coordinator: - error_code = error_codes_pb2.COORDINATOR_INTERNAL_ERROR + error_code = coordinator_pb2.COORDINATOR_INTERNAL_ERROR head, bodies = self._operation_executor.run_on_coordinator( dag, dag_bodies, loader_op_bodies ) diff --git a/docs/error_codes.md b/docs/error_codes.md new file mode 100644 index 000000000000..4c52a6a8851c --- /dev/null +++ b/docs/error_codes.md @@ -0,0 +1,3 @@ +# Error Codes + +The list of error codes is defined in protobuf. See [here](https://github.com/alibaba/GraphScope/tree/main/proto/error) for latest code collections. \ No newline at end of file diff --git a/flex/bin/interactive_server.cc b/flex/bin/interactive_server.cc index 32fc5e67e13a..ef7972887da5 100644 --- a/flex/bin/interactive_server.cc +++ b/flex/bin/interactive_server.cc @@ -171,7 +171,9 @@ int main(int argc, char** argv) { "start-compiler", bpo::value()->default_value(false), "whether or not to start compiler")( "memory-level,m", bpo::value()->default_value(1), - "memory allocation strategy"); + "memory allocation strategy")("enable-adhoc-handler", + bpo::value()->default_value(false), + "whether to enable adhoc handler"); setenv("TZ", "Asia/Shanghai", 1); tzset(); @@ -203,7 +205,7 @@ int main(int argc, char** argv) { service_config.start_admin_service = vm["enable-admin-service"].as(); service_config.start_compiler = vm["start-compiler"].as(); service_config.memory_level = vm["memory-level"].as(); - service_config.enable_adhoc_handler = true; + service_config.enable_adhoc_handler = vm["enable-adhoc-handler"].as(); auto& db = gs::GraphDB::get(); @@ -233,7 +235,9 @@ int main(int argc, char** argv) { LOG(INFO) << "Finish init workspace"; auto schema_file = server::WorkDirManipulator::GetGraphSchemaPath( service_config.default_graph); - gs::init_codegen_proxy(vm, engine_config_file); + if (service_config.enable_adhoc_handler) { + gs::init_codegen_proxy(vm, engine_config_file); + } } else { LOG(INFO) << "Start query service only"; std::string graph_schema_path, data_path; @@ -257,7 +261,9 @@ int main(int argc, char** argv) { } // The schema is loaded just to get the plugin dir and plugin list - gs::init_codegen_proxy(vm, engine_config_file, graph_schema_path); + if (service_config.enable_adhoc_handler) { + gs::init_codegen_proxy(vm, engine_config_file, graph_schema_path); + } db.Close(); auto load_res = db.Open(schema_res.value(), data_path, service_config.shard_num); diff --git a/flex/engines/graph_db/CMakeLists.txt b/flex/engines/graph_db/CMakeLists.txt index 6a9b0631745c..f280a20a5940 100644 --- a/flex/engines/graph_db/CMakeLists.txt +++ b/flex/engines/graph_db/CMakeLists.txt @@ -1,3 +1,4 @@ +add_subdirectory(runtime) file(GLOB_RECURSE GRAPH_DB_SRC_FILES "${CMAKE_CURRENT_SOURCE_DIR}/app/*.cc" "${CMAKE_CURRENT_SOURCE_DIR}/database/*.cc") @@ -6,6 +7,7 @@ add_library(flex_graph_db SHARED ${GRAPH_DB_SRC_FILES}) target_include_directories(flex_graph_db PUBLIC $) target_link_libraries(flex_graph_db flex_rt_mutable_graph flex_utils ${GLOG_LIBRARIES} ${LIBGRAPELITE_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT}) target_link_libraries(flex_graph_db hqps_plan_proto) +target_link_libraries(flex_graph_db runtime_adhoc) install_flex_target(flex_graph_db) install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/database/graph_db.h @@ -26,5 +28,7 @@ install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/app/app_base.h DESTINATION include/flex/engines/graph_db/app) install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/app/hqps_app.h DESTINATION include/flex/engines/graph_db/app) +install(FILES ${CMAKE_CURRENT_SOURCE_DIR}/app/adhoc_app.h + DESTINATION include/flex/engines/graph_db/app) diff --git a/flex/engines/graph_db/app/adhoc_app.cc b/flex/engines/graph_db/app/adhoc_app.cc new file mode 100644 index 000000000000..e8dbb626c80e --- /dev/null +++ b/flex/engines/graph_db/app/adhoc_app.cc @@ -0,0 +1,32 @@ +#include "flex/engines/graph_db/app/adhoc_app.h" + +#include "flex/engines/graph_db/runtime/adhoc/operators/operators.h" +#include "flex/engines/graph_db/runtime/adhoc/runtime.h" + +#include "flex/proto_generated_gie/physical.pb.h" + +namespace gs { + +bool AdhocReadApp::Query(const GraphDBSession& graph, Decoder& input, + Encoder& output) { + auto txn = graph.GetReadTransaction(); + + std::string_view plan_str = input.get_bytes(); + physical::PhysicalPlan plan; + if (!plan.ParseFromString(std::string(plan_str))) { + LOG(ERROR) << "Parse plan failed..."; + return false; + } + + LOG(INFO) << "plan: " << plan.DebugString(); + + auto ctx = runtime::runtime_eval(plan, txn, {}); + + runtime::eval_sink(ctx, txn, output); + + return true; +} +AppWrapper AdhocReadAppFactory::CreateApp(const GraphDB& db) { + return AppWrapper(new AdhocReadApp(), NULL); +} +} // namespace gs \ No newline at end of file diff --git a/flex/engines/graph_db/app/adhoc_app.h b/flex/engines/graph_db/app/adhoc_app.h new file mode 100644 index 000000000000..4e7b4a39de3c --- /dev/null +++ b/flex/engines/graph_db/app/adhoc_app.h @@ -0,0 +1,41 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ENGINES_GRAPH_DB_ADHOC_APP_H_ +#define ENGINES_GRAPH_DB_ADHOC_APP_H_ +#include "flex/engines/graph_db/app/app_base.h" +#include "flex/engines/graph_db/database/graph_db_session.h" + +namespace gs { +class AdhocReadApp : public ReadAppBase { + public: + AdhocReadApp() = default; + + AppType type() const override { return AppType::kCypherAdhoc; } + + bool Query(const GraphDBSession& graph, Decoder& input, + Encoder& output) override; +}; + +class AdhocReadAppFactory : public AppFactoryBase { + public: + AdhocReadAppFactory() = default; + ~AdhocReadAppFactory() = default; + + AppWrapper CreateApp(const GraphDB& db) override; +}; + +} // namespace gs +#endif // ENGINES_GRAPH_DB_ADHOC_APP_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/database/graph_db.cc b/flex/engines/graph_db/database/graph_db.cc index 16ad5826dffd..44eb7853db23 100644 --- a/flex/engines/graph_db/database/graph_db.cc +++ b/flex/engines/graph_db/database/graph_db.cc @@ -14,6 +14,7 @@ */ #include "flex/engines/graph_db/database/graph_db.h" +#include "flex/engines/graph_db/app/adhoc_app.h" #include "flex/engines/graph_db/app/hqps_app.h" #include "flex/engines/graph_db/app/server_app.h" #include "flex/engines/graph_db/database/graph_db_session.h" @@ -408,6 +409,8 @@ void GraphDB::initApps( std::make_shared(); app_factories_[Schema::HQPS_ADHOC_WRITE_PLUGIN_ID] = std::make_shared(); + app_factories_[Schema::ADHOC_READ_PLUGIN_ID] = + std::make_shared(); size_t valid_plugins = 0; for (auto& path_and_index : plugins) { diff --git a/flex/engines/graph_db/database/read_transaction.cc b/flex/engines/graph_db/database/read_transaction.cc index 4c9541f53fea..723345c6d30b 100644 --- a/flex/engines/graph_db/database/read_transaction.cc +++ b/flex/engines/graph_db/database/read_transaction.cc @@ -30,6 +30,8 @@ void ReadTransaction::Commit() { release(); } void ReadTransaction::Abort() { release(); } +const MutablePropertyFragment& ReadTransaction::graph() const { return graph_; } + ReadTransaction::vertex_iterator::vertex_iterator( label_t label, vid_t cur, vid_t num, const MutablePropertyFragment& graph) : label_(label), cur_(cur), num_(num), graph_(graph) {} diff --git a/flex/engines/graph_db/database/read_transaction.h b/flex/engines/graph_db/database/read_transaction.h index 84fde5d3cf27..5a88b1807f7e 100644 --- a/flex/engines/graph_db/database/read_transaction.h +++ b/flex/engines/graph_db/database/read_transaction.h @@ -286,6 +286,13 @@ class ReadTransaction { void Abort(); + const MutablePropertyFragment& graph() const; + + const std::shared_ptr get_vertex_property_column( + uint8_t label, const std::string& col_name) const { + return graph_.get_vertex_table(label).get_column(col_name); + } + class vertex_iterator { public: vertex_iterator(label_t label, vid_t cur, vid_t num, diff --git a/flex/engines/graph_db/runtime/CMakeLists.txt b/flex/engines/graph_db/runtime/CMakeLists.txt new file mode 100644 index 000000000000..40e05ab80e78 --- /dev/null +++ b/flex/engines/graph_db/runtime/CMakeLists.txt @@ -0,0 +1,11 @@ +file(GLOB_RECURSE COMMON_SOURCES "common/*.cc") +add_library(runtime_common SHARED ${COMMON_SOURCES}) +target_link_libraries(runtime_common ${Boost_LIBRARIES} flex_rt_mutable_graph flex_utils hqps_plan_proto) +install_flex_target(runtime_common) + +file(GLOB_RECURSE ADHOC_SOURCES "adhoc/*.cc") +add_library(runtime_adhoc SHARED ${ADHOC_SOURCES}) +target_link_libraries(runtime_adhoc runtime_common) +install_flex_target(runtime_adhoc) + + diff --git a/flex/engines/graph_db/runtime/adhoc/expr.cc b/flex/engines/graph_db/runtime/adhoc/expr.cc new file mode 100644 index 000000000000..648f7a108390 --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/expr.cc @@ -0,0 +1,58 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/adhoc/expr.h" + +namespace gs { + +namespace runtime { + +Expr::Expr(const ReadTransaction& txn, const Context& ctx, + const std::map& params, + const common::Expression& expr, VarType var_type) { + expr_ = parse_expression(txn, ctx, params, expr, var_type); +} + +RTAny Expr::eval_path(size_t idx) const { + RTAny ret = expr_->eval_path(idx); + return ret; +} + +RTAny Expr::eval_vertex(label_t label, vid_t v, size_t idx) const { + return expr_->eval_vertex(label, v, idx); +} +RTAny Expr::eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const { + return expr_->eval_edge(label, src, dst, data, idx); +} + +RTAny Expr::eval_path(size_t idx, int) const { + return expr_->eval_path(idx, 0); +} + +RTAny Expr::eval_vertex(label_t label, vid_t v, size_t idx, int) const { + return expr_->eval_vertex(label, v, idx, 0); +} + +RTAny Expr::eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx, int) const { + return expr_->eval_edge(label, src, dst, data, idx, 0); +} + +RTAnyType Expr::type() const { return expr_->type(); } + +} // namespace runtime + +} // namespace gs \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/adhoc/expr.h b/flex/engines/graph_db/runtime/adhoc/expr.h new file mode 100644 index 000000000000..13cc2eb2d1f0 --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/expr.h @@ -0,0 +1,58 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RUNTIME_ADHOC_RUNTIME_EXPR_H_ +#define RUNTIME_ADHOC_RUNTIME_EXPR_H_ + +#include "flex/engines/graph_db/database/read_transaction.h" + +#include "flex/engines/graph_db/runtime/adhoc/expr_impl.h" +#include "flex/engines/graph_db/runtime/common/rt_any.h" + +namespace gs { + +namespace runtime { + +class Expr { + public: + Expr(const ReadTransaction& txn, const Context& ctx, + const std::map& params, + const common::Expression& expr, VarType var_type); + + RTAny eval_path(size_t idx) const; + RTAny eval_vertex(label_t label, vid_t v, size_t idx) const; + RTAny eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const; + RTAny eval_path(size_t idx, int) const; + RTAny eval_vertex(label_t label, vid_t v, size_t idx, int) const; + RTAny eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx, int) const; + + RTAnyType type() const; + + std::shared_ptr builder() const { + return expr_->builder(); + } + + bool is_optional() const { return expr_->is_optional(); } + + private: + std::unique_ptr expr_; +}; + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_ADHOC_RUNTIME_EXPR_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/adhoc/expr_impl.cc b/flex/engines/graph_db/runtime/adhoc/expr_impl.cc new file mode 100644 index 000000000000..6b46396dee2f --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/expr_impl.cc @@ -0,0 +1,706 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/adhoc/expr_impl.h" +#include +#include + +namespace gs { + +namespace runtime { + +VariableExpr::VariableExpr(const ReadTransaction& txn, const Context& ctx, + const common::Variable& pb, VarType var_type) + : var_(txn, ctx, pb, var_type) {} +RTAny VariableExpr::eval_path(size_t idx) const { return var_.get(idx); } +RTAny VariableExpr::eval_vertex(label_t label, vid_t v, size_t idx) const { + return var_.get_vertex(label, v, idx); +} +RTAny VariableExpr::eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const { + return var_.get_edge(label, src, dst, data, idx); +} + +RTAny VariableExpr::eval_path(size_t idx, int) const { + return var_.get(idx, 0); +} + +RTAny VariableExpr::eval_vertex(label_t label, vid_t v, size_t idx, int) const { + return var_.get_vertex(label, v, idx, 0); +} + +RTAny VariableExpr::eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx, int) const { + return var_.get_edge(label, src, dst, data, idx, 0); +} + +RTAnyType VariableExpr::type() const { return var_.type(); } + +LogicalExpr::LogicalExpr(std::unique_ptr&& lhs, + std::unique_ptr&& rhs, common::Logical logic) + : lhs_(std::move(lhs)), rhs_(std::move(rhs)), logic_(logic) {} + +RTAny LogicalExpr::eval_path(size_t idx) const { + if (logic_ == common::Logical::LT) { + bool ret = lhs_->eval_path(idx) < rhs_->eval_path(idx); + return RTAny::from_bool(ret); + } else if (logic_ == common::Logical::GT) { + bool ret = rhs_->eval_path(idx) < lhs_->eval_path(idx); + return RTAny::from_bool(ret); + } else if (logic_ == common::Logical::GE) { + bool ret = lhs_->eval_path(idx) < rhs_->eval_path(idx); + return RTAny::from_bool(!ret); + } else if (logic_ == common::Logical::LE) { + bool ret = rhs_->eval_path(idx) < lhs_->eval_path(idx); + return RTAny::from_bool(!ret); + } else if (logic_ == common::Logical::EQ) { + bool ret = (rhs_->eval_path(idx) == lhs_->eval_path(idx)); + return RTAny::from_bool(ret); + } else if (logic_ == common::Logical::NE) { + bool ret = (rhs_->eval_path(idx) == lhs_->eval_path(idx)); + return RTAny::from_bool(!ret); + } else if (logic_ == common::Logical::AND) { + bool ret = + (rhs_->eval_path(idx).as_bool() && lhs_->eval_path(idx).as_bool()); + return RTAny::from_bool(ret); + } else if (logic_ == common::Logical::OR) { + bool ret = + (rhs_->eval_path(idx).as_bool() || lhs_->eval_path(idx).as_bool()); + return RTAny::from_bool(ret); + } else { + LOG(FATAL) << "not support..." << static_cast(logic_); + } + return RTAny::from_bool(false); +} + +RTAny LogicalExpr::eval_vertex(label_t label, vid_t v, size_t idx) const { + if (logic_ == common::Logical::LT) { + bool ret = + lhs_->eval_vertex(label, v, idx) < rhs_->eval_vertex(label, v, idx); + return RTAny::from_bool(ret); + } else if (logic_ == common::Logical::GT) { + bool ret = + rhs_->eval_vertex(label, v, idx) < lhs_->eval_vertex(label, v, idx); + return RTAny::from_bool(ret); + } else if (logic_ == common::Logical::GE) { + bool ret = + lhs_->eval_vertex(label, v, idx) < rhs_->eval_vertex(label, v, idx); + return RTAny::from_bool(!ret); + } else if (logic_ == common::Logical::LE) { + bool ret = + rhs_->eval_vertex(label, v, idx) < lhs_->eval_vertex(label, v, idx); + return RTAny::from_bool(!ret); + } else if (logic_ == common::Logical::EQ) { + bool ret = + (rhs_->eval_vertex(label, v, idx) == lhs_->eval_vertex(label, v, idx)); + return RTAny::from_bool(ret); + } else if (logic_ == common::Logical::NE) { + bool ret = + (rhs_->eval_vertex(label, v, idx) == lhs_->eval_vertex(label, v, idx)); + return RTAny::from_bool(!ret); + } else if (logic_ == common::Logical::AND) { + bool ret = (rhs_->eval_vertex(label, v, idx).as_bool() && + lhs_->eval_vertex(label, v, idx).as_bool()); + return RTAny::from_bool(ret); + } else if (logic_ == common::Logical::REGEX) { + std::string ret(lhs_->eval_vertex(label, v, idx).as_string()); + std::string rhs(rhs_->eval_vertex(label, v, idx).as_string()); + return RTAny::from_bool(std::regex_match(ret, std::regex(rhs))); + + } else { + LOG(FATAL) << "not support..." << static_cast(logic_); + } + return RTAny::from_bool(false); +} + +RTAny LogicalExpr::eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const { + if (logic_ == common::Logical::LT) { + bool ret = lhs_->eval_edge(label, src, dst, data, idx) < + rhs_->eval_edge(label, src, dst, data, idx); + return RTAny::from_bool(ret); + } else if (logic_ == common::Logical::GT) { + bool ret = rhs_->eval_edge(label, src, dst, data, idx) < + lhs_->eval_edge(label, src, dst, data, idx); + return RTAny::from_bool(ret); + } else if (logic_ == common::Logical::GE) { + bool ret = lhs_->eval_edge(label, src, dst, data, idx) < + rhs_->eval_edge(label, src, dst, data, idx); + return RTAny::from_bool(!ret); + } else if (logic_ == common::Logical::LE) { + bool ret = rhs_->eval_edge(label, src, dst, data, idx) < + lhs_->eval_edge(label, src, dst, data, idx); + return RTAny::from_bool(!ret); + } else if (logic_ == common::Logical::EQ) { + bool ret = (rhs_->eval_edge(label, src, dst, data, idx) == + lhs_->eval_edge(label, src, dst, data, idx)); + return RTAny::from_bool(ret); + } else if (logic_ == common::Logical::NE) { + bool ret = (rhs_->eval_edge(label, src, dst, data, idx) == + lhs_->eval_edge(label, src, dst, data, idx)); + return RTAny::from_bool(!ret); + } else if (logic_ == common::Logical::AND) { + bool ret = (rhs_->eval_edge(label, src, dst, data, idx).as_bool() && + lhs_->eval_edge(label, src, dst, data, idx).as_bool()); + return RTAny::from_bool(ret); + } else { + LOG(FATAL) << "not support..." << static_cast(logic_); + } + return RTAny::from_bool(false); +} + +RTAnyType LogicalExpr::type() const { return RTAnyType::kBoolValue; } + +UnaryLogicalExpr::UnaryLogicalExpr(std::unique_ptr&& expr, + common::Logical logic) + : expr_(std::move(expr)), logic_(logic) {} + +RTAny UnaryLogicalExpr::eval_path(size_t idx) const { + if (logic_ == common::Logical::NOT) { + return RTAny::from_bool(!expr_->eval_path(idx).as_bool()); + } else if (logic_ == common::Logical::ISNULL) { + return RTAny::from_bool(expr_->eval_path(idx, 0).type() == + RTAnyType::kNull); + } + LOG(FATAL) << "not support" << static_cast(logic_); + return RTAny::from_bool(false); +} + +RTAny UnaryLogicalExpr::eval_vertex(label_t label, vid_t v, size_t idx) const { + if (logic_ == common::Logical::NOT) { + return RTAny::from_bool(!expr_->eval_vertex(label, v, idx).as_bool()); + } else if (logic_ == common::Logical::ISNULL) { + return RTAny::from_bool(expr_->eval_vertex(label, v, idx, 0).is_null()); + } + LOG(FATAL) << "not support" << static_cast(logic_); + return RTAny::from_bool(false); +} + +RTAny UnaryLogicalExpr::eval_edge(const LabelTriplet& label, vid_t src, + vid_t dst, const Any& data, + size_t idx) const { + if (logic_ == common::Logical::NOT) { + return RTAny::from_bool( + !expr_->eval_edge(label, src, dst, data, idx).as_bool()); + } + LOG(FATAL) << "not support" << static_cast(logic_); + return RTAny::from_bool(false); +} + +RTAnyType UnaryLogicalExpr::type() const { return RTAnyType::kBoolValue; } + +ArithExpr::ArithExpr(std::unique_ptr&& lhs, + std::unique_ptr&& rhs, common::Arithmetic arith) + : lhs_(std::move(lhs)), rhs_(std::move(rhs)), arith_(arith) {} + +RTAny ArithExpr::eval_path(size_t idx) const { + switch (arith_) { + case common::Arithmetic::ADD: + return lhs_->eval_path(idx) + rhs_->eval_path(idx); + case common::Arithmetic::SUB: + return lhs_->eval_path(idx) - rhs_->eval_path(idx); + case common::Arithmetic::DIV: + return lhs_->eval_path(idx) / rhs_->eval_path(idx); + default: + LOG(FATAL) << "not support" << static_cast(arith_); + } + return RTAny(); +} + +RTAny ArithExpr::eval_vertex(label_t label, vid_t v, size_t idx) const { + switch (arith_) { + case common::Arithmetic::ADD: + return lhs_->eval_path(idx) + rhs_->eval_path(idx); + default: + LOG(FATAL) << "not support"; + } + return RTAny(); +} + +RTAny ArithExpr::eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const { + switch (arith_) { + case common::Arithmetic::ADD: + return lhs_->eval_path(idx) + rhs_->eval_path(idx); + default: + LOG(FATAL) << "not support"; + } + return RTAny(); +} + +RTAnyType ArithExpr::type() const { return lhs_->type(); } + +ConstExpr::ConstExpr(const RTAny& val) : val_(val) { + if (val_.type() == RTAnyType::kStringValue) { + s = val_.as_string(); + val_ = RTAny::from_string(s); + } +} +RTAny ConstExpr::eval_path(size_t idx) const { return val_; } +RTAny ConstExpr::eval_vertex(label_t label, vid_t v, size_t idx) const { + return val_; +} +RTAny ConstExpr::eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const { + return val_; +} + +RTAnyType ConstExpr::type() const { return val_.type(); } + +ExtractExpr::ExtractExpr(std::unique_ptr&& expr, + const common::Extract& extract) + : expr_(std::move(expr)), extract_(extract) {} + +static int32_t extract_year(int64_t ms) { + auto micro_second = ms / 1000; + struct tm tm; + gmtime_r((time_t*) (µ_second), &tm); + return tm.tm_year + 1900; +} + +static int32_t extract_month(int64_t ms) { + auto micro_second = ms / 1000; + struct tm tm; + gmtime_r((time_t*) (µ_second), &tm); + return tm.tm_mon + 1; +} + +static int32_t extract_day(int64_t ms) { + auto micro_second = ms / 1000; + struct tm tm; + gmtime_r((time_t*) (µ_second), &tm); + return tm.tm_mday; +} + +static int32_t extract_time_from_milli_second(int64_t ms, + common::Extract extract) { + if (extract.interval() == common::Extract::YEAR) { + return extract_year(ms); + } else if (extract.interval() == common::Extract::MONTH) { + return extract_month(ms); + } else if (extract.interval() == common::Extract::DAY) { + return extract_day(ms); + } else { + LOG(FATAL) << "not support"; + } + return 0; +} + +RTAny ExtractExpr::eval_path(size_t idx) const { + auto ms = expr_->eval_path(idx).as_date32(); + int32_t val = extract_time_from_milli_second(ms, extract_); + return RTAny::from_int32(val); +} + +RTAny ExtractExpr::eval_vertex(label_t label, vid_t v, size_t idx) const { + auto ms = expr_->eval_vertex(label, v, idx).as_date32(); + int32_t val = extract_time_from_milli_second(ms, extract_); + return RTAny::from_int32(val); +} + +RTAny ExtractExpr::eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const { + auto ms = expr_->eval_edge(label, src, dst, data, idx).as_date32(); + int32_t val = extract_time_from_milli_second(ms, extract_); + return RTAny::from_int32(val); +} + +RTAnyType ExtractExpr::type() const { return RTAnyType::kI32Value; } + +CaseWhenExpr::CaseWhenExpr( + std::vector, + std::unique_ptr>>&& when_then_exprs, + std::unique_ptr&& else_expr) + : when_then_exprs_(std::move(when_then_exprs)), + else_expr_(std::move(else_expr)) {} + +RTAny CaseWhenExpr::eval_path(size_t idx) const { + for (auto& pair : when_then_exprs_) { + if (pair.first->eval_path(idx).as_bool()) { + return pair.second->eval_path(idx); + } + } + return else_expr_->eval_path(idx); +} + +RTAny CaseWhenExpr::eval_vertex(label_t label, vid_t v, size_t idx) const { + for (auto& pair : when_then_exprs_) { + if (pair.first->eval_vertex(label, v, idx).as_bool()) { + return pair.second->eval_vertex(label, v, idx); + } + } + return else_expr_->eval_vertex(label, v, idx); +} + +RTAny CaseWhenExpr::eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const { + for (auto& pair : when_then_exprs_) { + if (pair.first->eval_edge(label, src, dst, data, idx).as_bool()) { + return pair.second->eval_edge(label, src, dst, data, idx); + } + } + return else_expr_->eval_edge(label, src, dst, data, idx); +} + +RTAnyType CaseWhenExpr::type() const { + RTAnyType type; + bool null_able = false; + if (when_then_exprs_.size() > 0) { + if (when_then_exprs_[0].second->type() == RTAnyType::kNull) { + null_able = true; + } else { + type = when_then_exprs_[0].second->type(); + } + } + if (else_expr_->type() == RTAnyType::kNull) { + null_able = true; + } else { + type = else_expr_->type(); + } + type.null_able_ = null_able; + return type; +} + +TupleExpr::TupleExpr(std::vector>&& exprs) + : exprs_(std::move(exprs)) {} + +RTAny TupleExpr::eval_path(size_t idx) const { + std::vector ret; + for (auto& expr : exprs_) { + ret.push_back(expr->eval_path(idx)); + } + return RTAny::from_tuple(std::move(ret)); +} + +RTAny TupleExpr::eval_vertex(label_t label, vid_t v, size_t idx) const { + std::vector ret; + for (auto& expr : exprs_) { + ret.push_back(expr->eval_vertex(label, v, idx)); + } + return RTAny::from_tuple(std::move(ret)); +} + +RTAny TupleExpr::eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const { + std::vector ret; + for (auto& expr : exprs_) { + ret.push_back(expr->eval_edge(label, src, dst, data, idx)); + } + return RTAny::from_tuple(std::move(ret)); +} + +RTAnyType TupleExpr::type() const { return RTAnyType::kTuple; } + +static RTAny parse_const_value(const common::Value& val) { + switch (val.item_case()) { + case common::Value::kI32: + return RTAny::from_int32(val.i32()); + case common::Value::kStr: + return RTAny::from_string(val.str()); + case common::Value::kI64: + return RTAny::from_int64(val.i64()); + case common::Value::kBoolean: + return RTAny::from_bool(val.boolean()); + case common::Value::kNone: + return RTAny(RTAnyType::kNull); + case common::Value::kF64: + return RTAny::from_double(val.f64()); + default: + LOG(FATAL) << "not support for " << val.item_case(); + } + return RTAny(); +} + +static RTAny parse_param(const common::DynamicParam& param, + const std::map& input) { + if (param.data_type().type_case() == + common::IrDataType::TypeCase::kDataType) { + common::DataType dt = param.data_type().data_type(); + const std::string& name = param.name(); + if (dt == common::DataType::DATE32) { + int64_t val = std::stoll(input.at(name)); + return RTAny::from_int64(val); + } else if (dt == common::DataType::STRING) { + const std::string& val = input.at(name); + return RTAny::from_string(val); + } else if (dt == common::DataType::INT32) { + int val = std::stoi(input.at(name)); + return RTAny::from_int32(val); + } + + LOG(FATAL) << "not support type: " << common::DataType_Name(dt); + } + LOG(FATAL) << "graph data type not expected...."; + return RTAny(); +} + +static inline int get_proiority(const common::ExprOpr& opr) { + switch (opr.item_case()) { + case common::ExprOpr::kBrace: { + return 17; + } + case common::ExprOpr::kExtract: { + return 2; + } + case common::ExprOpr::kLogical: { + switch (opr.logical()) { + case common::Logical::AND: + return 11; + case common::Logical::OR: + return 12; + case common::Logical::NOT: + return 2; + case common::Logical::WITHIN: + case common::Logical::WITHOUT: + return 2; + case common::Logical::EQ: + case common::Logical::NE: + return 7; + case common::Logical::GE: + case common::Logical::GT: + case common::Logical::LT: + case common::Logical::LE: + return 6; + case common::Logical::REGEX: + return 2; + default: + return 16; + } + } + case common::ExprOpr::kArith: { + switch (opr.arith()) { + case common::Arithmetic::ADD: + case common::Arithmetic::SUB: + return 4; + case common::Arithmetic::MUL: + case common::Arithmetic::DIV: + case common::Arithmetic::MOD: + return 3; + default: + return 16; + } + } + default: + return 16; + } + return 16; +} +static std::unique_ptr parse_expression_impl( + const ReadTransaction& txn, const Context& ctx, + const std::map& params, + const common::Expression& expr, VarType var_type); +static std::unique_ptr build_expr( + const ReadTransaction& txn, const Context& ctx, + const std::map& params, + std::stack& opr_stack, VarType var_type) { + while (!opr_stack.empty()) { + auto opr = opr_stack.top(); + opr_stack.pop(); + switch (opr.item_case()) { + case common::ExprOpr::kConst: { + if (opr.const_().item_case() == common::Value::kStr) { + const std::string& str = opr.const_().str(); + return std::make_unique(RTAny::from_string(str)); + } + return std::make_unique(parse_const_value(opr.const_())); + } + case common::ExprOpr::kParam: { + return std::make_unique(parse_param(opr.param(), params)); + } + case common::ExprOpr::kVar: { + return std::make_unique(txn, ctx, opr.var(), var_type); + } + case common::ExprOpr::kLogical: { + if (opr.logical() == common::Logical::WITHIN) { + auto lhs = opr_stack.top(); + opr_stack.pop(); + auto rhs = opr_stack.top(); + opr_stack.pop(); + CHECK(lhs.has_var()); + CHECK(rhs.has_const_()); + auto key = + std::make_unique(txn, ctx, lhs.var(), var_type); + if (key->type() == RTAnyType::kI64Value) { + return std::make_unique>(txn, ctx, std::move(key), + rhs.const_()); + } else if (key->type() == RTAnyType::kI32Value) { + return std::make_unique>(txn, ctx, std::move(key), + rhs.const_()); + } else { + LOG(FATAL) << "not support"; + } + } else if (opr.logical() == common::Logical::NOT || + opr.logical() == common::Logical::ISNULL) { + auto lhs = build_expr(txn, ctx, params, opr_stack, var_type); + return std::make_unique(std::move(lhs), + opr.logical()); + } else { + auto lhs = build_expr(txn, ctx, params, opr_stack, var_type); + auto rhs = build_expr(txn, ctx, params, opr_stack, var_type); + return std::make_unique(std::move(lhs), std::move(rhs), + opr.logical()); + } + break; + } + case common::ExprOpr::kArith: { + auto lhs = build_expr(txn, ctx, params, opr_stack, var_type); + auto rhs = build_expr(txn, ctx, params, opr_stack, var_type); + return std::make_unique(std::move(lhs), std::move(rhs), + opr.arith()); + } + case common::ExprOpr::kCase: { + auto op = opr.case_(); + size_t len = op.when_then_expressions_size(); + std::vector< + std::pair, std::unique_ptr>> + when_then_exprs; + for (size_t i = 0; i < len; ++i) { + auto when_expr = op.when_then_expressions(i).when_expression(); + auto then_expr = op.when_then_expressions(i).then_result_expression(); + when_then_exprs.emplace_back( + parse_expression_impl(txn, ctx, params, when_expr, var_type), + parse_expression_impl(txn, ctx, params, then_expr, var_type)); + } + auto else_expr = parse_expression_impl( + txn, ctx, params, op.else_result_expression(), var_type); + return std::make_unique(std::move(when_then_exprs), + std::move(else_expr)); + } + case common::ExprOpr::kExtract: { + auto hs = build_expr(txn, ctx, params, opr_stack, var_type); + return std::make_unique(std::move(hs), opr.extract()); + } + case common::ExprOpr::kVars: { + auto op = opr.vars(); + std::vector> exprs; + for (int i = 0; i < op.keys_size(); ++i) { + exprs.push_back( + std::make_unique(txn, ctx, op.keys(i), var_type)); + } + return std::make_unique(std::move(exprs)); + // LOG(FATAL) << "not support" << opr.DebugString(); + // break; + } + case common::ExprOpr::kMap: { + auto op = opr.map(); + std::vector keys_vec; + std::vector> exprs; + for (int i = 0; i < op.key_vals_size(); ++i) { + auto key = op.key_vals(i).key(); + auto val = op.key_vals(i).val(); + auto any = parse_const_value(key); + CHECK(any.type() == RTAnyType::kStringValue); + { + auto str = any.as_string(); + keys_vec.push_back(std::string(str)); + } + exprs.emplace_back( + std::make_unique(txn, ctx, val, + var_type)); // just for parse + } + if (exprs.size() > 0) { + return std::make_unique(std::move(keys_vec), std::move(exprs)); + } + LOG(FATAL) << "not support" << opr.DebugString(); + } + default: + LOG(FATAL) << "not support" << opr.DebugString(); + break; + } + } + return nullptr; +} +static std::unique_ptr parse_expression_impl( + const ReadTransaction& txn, const Context& ctx, + const std::map& params, + const common::Expression& expr, VarType var_type) { + std::stack opr_stack; + std::stack opr_stack2; + const auto& oprs = expr.operators(); + for (auto it = oprs.rbegin(); it != oprs.rend(); ++it) { + switch ((*it).item_case()) { + case common::ExprOpr::kBrace: { + auto brace = (*it).brace(); + if (brace == common::ExprOpr::Brace::ExprOpr_Brace_LEFT_BRACE) { + while (!opr_stack.empty() && + opr_stack.top().item_case() != common::ExprOpr::kBrace) { + opr_stack2.push(opr_stack.top()); + opr_stack.pop(); + } + CHECK(!opr_stack.empty()); + opr_stack.pop(); + } else if (brace == common::ExprOpr::Brace::ExprOpr_Brace_RIGHT_BRACE) { + opr_stack.emplace(*it); + } + break; + } + case common::ExprOpr::kConst: + case common::ExprOpr::kVar: + case common::ExprOpr::kParam: + case common::ExprOpr::kVars: { + opr_stack2.push(*it); + break; + } + case common::ExprOpr::kArith: + case common::ExprOpr::kLogical: { + // unary operator + if ((*it).logical() == common::Logical::NOT || + (*it).logical() == common::Logical::ISNULL) { + opr_stack2.push(*it); + break; + } + + while (!opr_stack.empty() && + get_proiority(opr_stack.top()) <= get_proiority(*it)) { + opr_stack2.push(opr_stack.top()); + opr_stack.pop(); + } + opr_stack.push(*it); + break; + } + case common::ExprOpr::kExtract: { + opr_stack2.push(*it); + break; + } + case common::ExprOpr::kCase: { + opr_stack2.push(*it); + break; + } + case common::ExprOpr::kMap: { + opr_stack2.push(*it); + break; + } + default: { + LOG(FATAL) << "not support" << (*it).DebugString(); + break; + } + } + } + while (!opr_stack.empty()) { + opr_stack2.push(opr_stack.top()); + opr_stack.pop(); + } + return build_expr(txn, ctx, params, opr_stack2, var_type); +} +std::unique_ptr parse_expression( + const ReadTransaction& txn, const Context& ctx, + const std::map& params, + const common::Expression& expr, VarType var_type) { + return parse_expression_impl(txn, ctx, params, expr, var_type); +} + +} // namespace runtime + +} // namespace gs diff --git a/flex/engines/graph_db/runtime/adhoc/expr_impl.h b/flex/engines/graph_db/runtime/adhoc/expr_impl.h new file mode 100644 index 000000000000..a3d5af79bfc0 --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/expr_impl.h @@ -0,0 +1,387 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RUNTIME_ADHOC_RUNTIME_EXPR_IMPL_H_ +#define RUNTIME_ADHOC_RUNTIME_EXPR_IMPL_H_ + +#include "flex/proto_generated_gie/expr.pb.h" + +#include "flex/engines/graph_db/runtime/adhoc/var.h" +#include "flex/engines/graph_db/runtime/common/rt_any.h" + +namespace gs { + +namespace runtime { + +class ExprBase { + public: + virtual RTAny eval_path(size_t idx) const = 0; + virtual RTAny eval_vertex(label_t label, vid_t v, size_t idx) const = 0; + virtual RTAny eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const = 0; + virtual RTAnyType type() const = 0; + virtual RTAny eval_path(size_t idx, int) const { return eval_path(idx); } + virtual RTAny eval_vertex(label_t label, vid_t v, size_t idx, int) const { + return eval_vertex(label, v, idx); + } + virtual RTAny eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx, int) const { + return eval_edge(label, src, dst, data, idx); + } + virtual std::shared_ptr builder() const { + LOG(FATAL) << "not implemented"; + return nullptr; + } + + virtual bool is_optional() const { return false; } + + virtual ~ExprBase() = default; +}; + +class ConstTrueExpr : public ExprBase { + public: + RTAny eval_path(size_t idx) const override { return RTAny::from_bool(true); } + RTAny eval_vertex(label_t label, vid_t v, size_t idx) const override { + return RTAny::from_bool(true); + } + + RTAny eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const override { + return RTAny::from_bool(true); + } + + RTAnyType type() const override { return RTAnyType::kBoolValue; } +}; + +class ConstFalseExpr : public ExprBase { + public: + RTAny eval_path(size_t idx) const override { return RTAny::from_bool(false); } + RTAny eval_vertex(label_t label, vid_t v, size_t idx) const override { + return RTAny::from_bool(false); + } + RTAny eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const override { + return RTAny::from_bool(false); + } + + RTAnyType type() const override { return RTAnyType::kBoolValue; } +}; + +template +class WithInExpr : public ExprBase { + public: + WithInExpr(const ReadTransaction& txn, const Context& ctx, + std::unique_ptr&& key, const common::Value& array) + : key_(std::move(key)) { + if constexpr (std::is_same_v) { + CHECK(array.item_case() == common::Value::kI64Array); + size_t len = array.i64_array().item_size(); + for (size_t idx = 0; idx < len; ++idx) { + container_.push_back(array.i64_array().item(idx)); + } + } else if constexpr (std::is_same_v) { + CHECK(array.item_case() == common::Value::kI32Array); + size_t len = array.i32_array().item_size(); + for (size_t idx = 0; idx < len; ++idx) { + container_.push_back(array.i32_array().item(idx)); + } + } else { + LOG(FATAL) << "not implemented"; + } + } + + RTAny eval_path(size_t idx) const override { + auto val = TypedConverter::to_typed(key_->eval_path(idx)); + return RTAny::from_bool(std::find(container_.begin(), container_.end(), + val) != container_.end()); + } + + RTAny eval_path(size_t idx, int) const override { + auto any_val = key_->eval_path(idx, 0); + if (any_val.is_null()) { + return RTAny::from_bool(false); + } + auto val = TypedConverter::to_typed(any_val); + return RTAny::from_bool(std::find(container_.begin(), container_.end(), + val) != container_.end()); + } + RTAny eval_vertex(label_t label, vid_t v, size_t idx) const override { + auto val = TypedConverter::to_typed(key_->eval_vertex(label, v, idx)); + return RTAny::from_bool(std::find(container_.begin(), container_.end(), + val) != container_.end()); + } + + RTAny eval_vertex(label_t label, vid_t v, size_t idx, int) const override { + auto any_val = key_->eval_vertex(label, v, idx, 0); + if (any_val.is_null()) { + return RTAny::from_bool(false); + } + auto val = TypedConverter::to_typed(any_val); + return RTAny::from_bool(std::find(container_.begin(), container_.end(), + val) != container_.end()); + } + + RTAny eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const override { + LOG(FATAL) << "not implemented"; + return RTAny::from_bool(false); + } + RTAnyType type() const override { return RTAnyType::kBoolValue; } + bool is_optional() const override { return key_->is_optional(); } + + std::unique_ptr key_; + std::vector container_; +}; + +class VariableExpr : public ExprBase { + public: + VariableExpr(const ReadTransaction& txn, const Context& ctx, + const common::Variable& pb, VarType var_type); + + RTAny eval_path(size_t idx) const override; + RTAny eval_vertex(label_t label, vid_t v, size_t idx) const override; + RTAny eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const override; + RTAnyType type() const override; + + RTAny eval_path(size_t idx, int) const override; + RTAny eval_vertex(label_t label, vid_t v, size_t idx, int) const override; + RTAny eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx, int) const override; + + std::shared_ptr builder() const override { + return var_.builder(); + } + + bool is_optional() const override { return var_.is_optional(); } + + private: + Var var_; +}; + +class UnaryLogicalExpr : public ExprBase { + public: + UnaryLogicalExpr(std::unique_ptr&& expr, common::Logical logic); + + RTAny eval_path(size_t idx) const override; + RTAny eval_vertex(label_t label, vid_t v, size_t idx) const override; + RTAny eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const override; + + RTAnyType type() const override; + + bool is_optional() const override { return expr_->is_optional(); } + + private: + std::unique_ptr expr_; + common::Logical logic_; +}; +class LogicalExpr : public ExprBase { + public: + LogicalExpr(std::unique_ptr&& lhs, std::unique_ptr&& rhs, + common::Logical logic); + + RTAny eval_path(size_t idx) const override; + RTAny eval_vertex(label_t label, vid_t v, size_t idx) const override; + RTAny eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const override; + + RTAny eval_path(size_t idx, int) const override { + if (lhs_->eval_path(idx, 0).is_null() || + rhs_->eval_path(idx, 0).is_null()) { + return RTAny::from_bool(false); + } + return eval_path(idx); + } + RTAny eval_vertex(label_t label, vid_t v, size_t idx, int) const override { + if (lhs_->eval_vertex(label, v, idx, 0).is_null() || + rhs_->eval_vertex(label, v, idx, 0).is_null()) { + return RTAny::from_bool(false); + } + return eval_vertex(label, v, idx); + } + RTAny eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx, int) const override { + LOG(FATAL) << "not implemented"; + } + + RTAnyType type() const override; + + bool is_optional() const override { + return lhs_->is_optional() || rhs_->is_optional(); + } + + private: + std::unique_ptr lhs_; + std::unique_ptr rhs_; + common::Logical logic_; +}; + +class ExtractExpr : public ExprBase { + public: + ExtractExpr(std::unique_ptr&& expr, const common::Extract& extract); + + RTAny eval_path(size_t idx) const override; + RTAny eval_vertex(label_t label, vid_t v, size_t idx) const override; + RTAny eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const override; + + RTAnyType type() const override; + + private: + std::unique_ptr expr_; + const common::Extract extract_; +}; +class ArithExpr : public ExprBase { + public: + ArithExpr(std::unique_ptr&& lhs, std::unique_ptr&& rhs, + common::Arithmetic arith); + + RTAny eval_path(size_t idx) const override; + RTAny eval_vertex(label_t label, vid_t v, size_t idx) const override; + RTAny eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const override; + + RTAnyType type() const override; + + private: + std::unique_ptr lhs_; + std::unique_ptr rhs_; + common::Arithmetic arith_; +}; + +class ConstExpr : public ExprBase { + public: + ConstExpr(const RTAny& val); + RTAny eval_path(size_t idx) const override; + RTAny eval_vertex(label_t label, vid_t v, size_t idx) const override; + RTAny eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const override; + + RTAnyType type() const override; + + private: + RTAny val_; + std::string s; +}; + +class CaseWhenExpr : public ExprBase { + public: + CaseWhenExpr( + std::vector, + std::unique_ptr>>&& when_then_exprs, + std::unique_ptr&& else_expr); + + RTAny eval_path(size_t idx) const override; + RTAny eval_vertex(label_t label, vid_t v, size_t idx) const override; + RTAny eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const override; + + RTAnyType type() const override; + + private: + std::vector, std::unique_ptr>> + when_then_exprs_; + std::unique_ptr else_expr_; +}; + +class TupleExpr : public ExprBase { + public: + TupleExpr(std::vector>&& exprs); + + RTAny eval_path(size_t idx) const override; + RTAny eval_vertex(label_t label, vid_t v, size_t idx) const override; + RTAny eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const override; + + RTAnyType type() const override; + + private: + std::vector> exprs_; +}; + +class MapExpr : public ExprBase { + public: + MapExpr(std::vector&& keys, + std::vector>&& values) + : keys(std::move(keys)), value_exprs(std::move(values)) { + CHECK(keys.size() == values.size()); + } + + RTAny eval_path(size_t idx) const override { + std::vector ret; + for (size_t i = 0; i < keys.size(); i++) { + ret.push_back(value_exprs[i]->eval_path(idx)); + } + values.emplace_back(ret); + size_t id = values.size() - 1; + auto map_impl = MapImpl::make_map_impl(&keys, &values[id]); + auto map = Map::make_map(map_impl); + return RTAny::from_map(map); + } + + RTAny eval_path(size_t idx, int) const override { + std::vector ret; + for (size_t i = 0; i < keys.size(); i++) { + ret.push_back(value_exprs[i]->eval_path(idx, 0)); + } + values.emplace_back(ret); + size_t id = values.size() - 1; + auto map_impl = MapImpl::make_map_impl(&keys, &values[id]); + auto map = Map::make_map(map_impl); + return RTAny::from_map(map); + } + RTAny eval_vertex(label_t label, vid_t v, size_t idx) const override { + LOG(FATAL) << "not implemented"; + return RTAny(); + } + RTAny eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const override { + LOG(FATAL) << "not implemented"; + return RTAny(); + } + + RTAnyType type() const override { return RTAnyType::kMap; } + + bool is_optional() const override { + for (auto& expr : value_exprs) { + if (expr->is_optional()) { + return true; + } + } + return false; + } + + std::shared_ptr builder() const override { + auto builder = std::make_shared(); + builder->set_keys(keys); + return std::dynamic_pointer_cast(builder); + } + + private: + std::vector keys; + std::vector> value_exprs; + mutable std::vector> values; +}; +std::unique_ptr parse_expression( + const ReadTransaction& txn, const Context& ctx, + const std::map& params, + const common::Expression& expr, VarType var_type); + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_ADHOC_RUNTIME_EXPR_IMPL_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/adhoc/operators/dedup.cc b/flex/engines/graph_db/runtime/adhoc/operators/dedup.cc new file mode 100644 index 000000000000..8f3fcf92aae7 --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/operators/dedup.cc @@ -0,0 +1,55 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/common/operators/dedup.h" +#include "flex/engines/graph_db/runtime/adhoc/operators/operators.h" +#include "flex/engines/graph_db/runtime/adhoc/var.h" + +namespace gs { + +namespace runtime { + +Context eval_dedup(const algebra::Dedup& opr, const ReadTransaction& txn, + Context&& ctx) { + std::vector keys; + std::vector> vars; + int keys_num = opr.keys_size(); + bool flag = false; + for (int k_i = 0; k_i < keys_num; ++k_i) { + const common::Variable& key = opr.keys(k_i); + + int tag = -1; + if (key.has_tag()) { + tag = key.tag().id(); + } + if (key.has_property()) { + Var var(txn, ctx, key, VarType::kPathVar); + vars.emplace_back([var](size_t i) { return var.get(i); }); + flag = true; + } else { + keys.push_back(tag); + } + } + if (!flag) { + Dedup::dedup(txn, ctx, keys); + } else { + Dedup::dedup(txn, ctx, keys, vars); + } + return ctx; +} + +} // namespace runtime + +} // namespace gs diff --git a/flex/engines/graph_db/runtime/adhoc/operators/edge_expand.cc b/flex/engines/graph_db/runtime/adhoc/operators/edge_expand.cc new file mode 100644 index 000000000000..1f671b826c7c --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/operators/edge_expand.cc @@ -0,0 +1,95 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/common/operators/edge_expand.h" +#include "flex/engines/graph_db/runtime/adhoc/operators/operators.h" +#include "flex/engines/graph_db/runtime/adhoc/predicates.h" +#include "flex/engines/graph_db/runtime/adhoc/utils.h" +#include "flex/engines/graph_db/runtime/common/context.h" + +namespace gs { + +namespace runtime { + +Context eval_edge_expand(const physical::EdgeExpand& opr, + const ReadTransaction& txn, Context&& ctx, + const std::map& params, + const physical::PhysicalOpr_MetaData& meta) { + if (ctx.row_num() == 0) { + return ctx; + } + int v_tag; + if (!opr.has_v_tag()) { + v_tag = -1; + } else { + v_tag = opr.v_tag().value(); + } + + Direction dir = parse_direction(opr.direction()); + bool is_optional = opr.is_optional(); + CHECK(!is_optional); + + CHECK(opr.has_params()); + const algebra::QueryParams& query_params = opr.params(); + + int alias = -1; + if (opr.has_alias()) { + alias = opr.alias().value(); + } + + if (opr.expand_opt() == + physical::EdgeExpand_ExpandOpt::EdgeExpand_ExpandOpt_VERTEX) { + if (query_params.has_predicate()) { + LOG(FATAL) << "not support"; + } else { + EdgeExpandParams eep; + eep.v_tag = v_tag; + eep.labels = parse_label_triplets(meta); + eep.dir = dir; + eep.alias = alias; + return EdgeExpand::expand_vertex_without_predicate(txn, std::move(ctx), + eep); + } + } else if (opr.expand_opt() == + physical::EdgeExpand_ExpandOpt::EdgeExpand_ExpandOpt_EDGE) { + if (query_params.has_predicate()) { + EdgeExpandParams eep; + eep.v_tag = v_tag; + eep.labels = parse_label_triplets(meta); + eep.dir = dir; + eep.alias = alias; + + GeneralEdgePredicate pred(txn, ctx, params, query_params.predicate()); + + return EdgeExpand::expand_edge(txn, std::move(ctx), eep, pred); + } else { + EdgeExpandParams eep; + eep.v_tag = v_tag; + eep.labels = parse_label_triplets(meta); + eep.dir = dir; + eep.alias = alias; + + return EdgeExpand::expand_edge_without_predicate(txn, std::move(ctx), + eep); + } + } else { + LOG(FATAL) << "not support"; + } + return ctx; +} + +} // namespace runtime + +} // namespace gs \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/adhoc/operators/get_v.cc b/flex/engines/graph_db/runtime/adhoc/operators/get_v.cc new file mode 100644 index 000000000000..c964099e5101 --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/operators/get_v.cc @@ -0,0 +1,88 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/common/operators/get_v.h" +#include "flex/engines/graph_db/runtime/adhoc/operators/operators.h" +#include "flex/engines/graph_db/runtime/adhoc/predicates.h" +#include "flex/engines/graph_db/runtime/adhoc/utils.h" + +namespace gs { + +namespace runtime { + +VOpt parse_opt(const physical::GetV_VOpt& opt) { + if (opt == physical::GetV_VOpt::GetV_VOpt_START) { + return VOpt::kStart; + } else if (opt == physical::GetV_VOpt::GetV_VOpt_END) { + return VOpt::kEnd; + } else if (opt == physical::GetV_VOpt::GetV_VOpt_OTHER) { + return VOpt::kOther; + } else if (opt == physical::GetV_VOpt::GetV_VOpt_BOTH) { + return VOpt::kBoth; + } else if (opt == physical::GetV_VOpt::GetV_VOpt_ITSELF) { + return VOpt::kItself; + } else { + LOG(FATAL) << "unexpected GetV::Opt"; + return VOpt::kItself; + } +} + +Context eval_get_v(const physical::GetV& opr, const ReadTransaction& txn, + Context&& ctx, + const std::map& params) { + if (ctx.row_num() == 0) { + return ctx; + } + int tag = -1; + if (opr.has_tag()) { + tag = opr.tag().value(); + } + VOpt opt = parse_opt(opr.opt()); + int alias = -1; + if (opr.has_alias()) { + alias = opr.alias().value(); + } + + if (opr.has_params()) { + const algebra::QueryParams& query_params = opr.params(); + GetVParams p; + p.opt = opt; + p.tag = tag; + p.tables = parse_tables(query_params); + p.alias = alias; + if (query_params.has_predicate()) { + GeneralVertexPredicate pred(txn, ctx, params, query_params.predicate()); + + if (opt == VOpt::kItself) { + return GetV::get_vertex_from_vertices(txn, std::move(ctx), p, pred); + } else if (opt == VOpt::kEnd || opt == VOpt::kStart) { + return GetV::get_vertex_from_edges(txn, std::move(ctx), p, pred); + } + } else { + if (opt == VOpt::kEnd || opt == VOpt::kStart || opt == VOpt::kOther) { + auto ret = GetV::get_vertex_from_edges(txn, std::move(ctx), p, + DummyVertexPredicate()); + return ret; + } + } + } + + LOG(FATAL) << "not support"; + return ctx; +} + +} // namespace runtime + +} // namespace gs \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/adhoc/operators/group_by.cc b/flex/engines/graph_db/runtime/adhoc/operators/group_by.cc new file mode 100644 index 000000000000..aedf96058238 --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/operators/group_by.cc @@ -0,0 +1,559 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/adhoc/operators/operators.h" +#include "flex/engines/graph_db/runtime/adhoc/utils.h" +#include "flex/engines/graph_db/runtime/adhoc/var.h" +#include "flex/engines/graph_db/runtime/common/columns/value_columns.h" +#include "flex/engines/graph_db/runtime/common/columns/vertex_columns.h" + +namespace gs { + +namespace runtime { + +enum class AggrKind { + kSum, + kMin, + kMax, + kCount, + kCountDistinct, + kToSet, + kFirst, + kToList, + kAvg, +}; + +AggrKind parse_aggregate(physical::GroupBy_AggFunc::Aggregate v) { + if (v == physical::GroupBy_AggFunc::SUM) { + return AggrKind::kSum; + } else if (v == physical::GroupBy_AggFunc::MIN) { + return AggrKind::kMin; + } else if (v == physical::GroupBy_AggFunc::MAX) { + return AggrKind::kMax; + } else if (v == physical::GroupBy_AggFunc::COUNT) { + return AggrKind::kCount; + } else if (v == physical::GroupBy_AggFunc::COUNT_DISTINCT) { + return AggrKind::kCountDistinct; + } else if (v == physical::GroupBy_AggFunc::TO_SET) { + return AggrKind::kToSet; + } else if (v == physical::GroupBy_AggFunc::FIRST) { + return AggrKind::kFirst; + } else if (v == physical::GroupBy_AggFunc::TO_LIST) { + return AggrKind::kToList; + } else if (v == physical::GroupBy_AggFunc::AVG) { + return AggrKind::kAvg; + } else { + LOG(FATAL) << "unsupport" << static_cast(v); + return AggrKind::kSum; + } +} + +struct AggFunc { + AggFunc(const physical::GroupBy_AggFunc& opr, const ReadTransaction& txn, + const Context& ctx) + : aggregate(parse_aggregate(opr.aggregate())), alias(-1) { + if (opr.has_alias()) { + alias = opr.alias().value(); + } + int var_num = opr.vars_size(); + for (int i = 0; i < var_num; ++i) { + vars.emplace_back(txn, ctx, opr.vars(i), VarType::kPathVar); + } + } + + std::vector vars; + AggrKind aggregate; + int alias; +}; + +struct AggKey { + AggKey(const physical::GroupBy_KeyAlias& opr, const ReadTransaction& txn, + const Context& ctx) + : key(txn, ctx, opr.key(), VarType::kPathVar), alias(-1) { + if (opr.has_alias()) { + alias = opr.alias().value(); + } + } + + Var key; + int alias; + std::shared_ptr column_builder; +}; + +std::pair>, Context> generate_aggregate_indices( + const std::vector& keys, size_t row_num, + const std::vector& functions) { + std::unordered_map sig_to_root; + std::vector> root_list; + std::vector> ret; + + size_t keys_num = keys.size(); + std::vector> keys_columns; + std::vector keys_row(keys_num); + for (size_t k_i = 0; k_i < keys_num; ++k_i) { + auto type = keys[k_i].key.type(); + std::shared_ptr builder; + if (type == RTAnyType::kList) { + builder = keys[k_i].key.builder(); + } else { + builder = create_column_builder(type); + } + keys_columns.push_back(builder); + } + + for (size_t r_i = 0; r_i < row_num; ++r_i) { + bool has_null{false}; + for (auto func : functions) { + for (size_t v_i = 0; v_i < func.vars.size(); ++v_i) { + if (func.vars[v_i].is_optional()) { + if (func.vars[v_i].get(r_i, 0).is_null()) { + has_null = true; + break; + } + } + } + if (has_null) { + break; + } + } + + std::vector buf; + ::gs::Encoder encoder(buf); + for (size_t k_i = 0; k_i < keys_num; ++k_i) { + keys_row[k_i] = keys[k_i].key.get(r_i); + keys_row[k_i].encode_sig(keys[k_i].key.type(), encoder); + } + + std::string_view sv(buf.data(), buf.size()); + auto iter = sig_to_root.find(sv); + if (iter != sig_to_root.end()) { + if (!has_null) { + ret[iter->second].push_back(r_i); + } + } else { + sig_to_root.emplace(sv, ret.size()); + root_list.emplace_back(std::move(buf)); + + for (size_t k_i = 0; k_i < keys_num; ++k_i) { + keys_columns[k_i]->push_back_elem(keys_row[k_i]); + } + + std::vector ret_elem; + if (!has_null) { + ret_elem.push_back(r_i); + } + ret.emplace_back(std::move(ret_elem)); + } + } + + Context ret_ctx; + for (size_t k_i = 0; k_i < keys_num; ++k_i) { + ret_ctx.set(keys[k_i].alias, keys_columns[k_i]->finish()); + ret_ctx.append_tag_id(keys[k_i].alias); + } + + return std::make_pair(std::move(ret), std::move(ret_ctx)); +} + +template +std::shared_ptr numeric_sum( + const Var& var, const std::vector>& to_aggregate) { + ValueColumnBuilder builder; + size_t col_size = to_aggregate.size(); + builder.reserve(col_size); + for (size_t k = 0; k < col_size; ++k) { + auto& vec = to_aggregate[k]; + NT s = 0; + for (auto idx : vec) { + s += TypedConverter::to_typed(var.get(idx)); + } + builder.push_back_opt(s); + } + return builder.finish(); +} + +template +std::shared_ptr numeric_count_distinct( + const Var& var, const std::vector>& to_aggregate) { + ValueColumnBuilder builder; + size_t col_size = to_aggregate.size(); + builder.reserve(col_size); + for (size_t k = 0; k < col_size; ++k) { + auto& vec = to_aggregate[k]; + std::unordered_set s; + for (auto idx : vec) { + s.insert(TypedConverter::to_typed(var.get(idx))); + } + builder.push_back_opt(s.size()); + } + return builder.finish(); +} + +std::shared_ptr vertex_count_distinct( + const Var& var, const std::vector>& to_aggregate) { + ValueColumnBuilder builder; + size_t col_size = to_aggregate.size(); + builder.reserve(col_size); + for (size_t k = 0; k < col_size; ++k) { + auto& vec = to_aggregate[k]; + std::set> s; + for (auto idx : vec) { + s.insert(var.get(idx).as_vertex()); + } + builder.push_back_opt(s.size()); + } + return builder.finish(); +} + +std::shared_ptr general_count_distinct( + const std::vector& vars, + const std::vector>& to_aggregate) { + ValueColumnBuilder builder; + size_t col_size = to_aggregate.size(); + builder.reserve(col_size); + for (size_t k = 0; k < col_size; ++k) { + auto& vec = to_aggregate[k]; + std::set s; + std::vector bytes; + for (auto idx : vec) { + bytes.clear(); + Encoder encoder(bytes); + for (auto& var : vars) { + var.get(idx).encode_sig(var.get(idx).type(), encoder); + encoder.put_byte('#'); + } + std::string str(bytes.begin(), bytes.end()); + s.insert(str); + } + builder.push_back_opt(s.size()); + } + return builder.finish(); +} + +std::shared_ptr general_count( + const std::vector& vars, + const std::vector>& to_aggregate) { + ValueColumnBuilder builder; + if (vars.size() == 1) { + if (vars[0].is_optional()) { + size_t col_size = to_aggregate.size(); + builder.reserve(col_size); + for (size_t k = 0; k < col_size; ++k) { + auto& vec = to_aggregate[k]; + int64_t s = 0; + for (auto idx : vec) { + if (vars[0].get(idx, 0).is_null()) { + continue; + } + s += 1; + } + builder.push_back_opt(s); + } + return builder.finish(); + } + } + size_t col_size = to_aggregate.size(); + builder.reserve(col_size); + for (size_t k = 0; k < col_size; ++k) { + auto& vec = to_aggregate[k]; + builder.push_back_opt(vec.size()); + } + return builder.finish(); +} + +std::shared_ptr vertex_first( + const Var& var, const std::vector>& to_aggregate) { + MLVertexColumnBuilder builder; + size_t col_size = to_aggregate.size(); + builder.reserve(col_size); + for (size_t k = 0; k < col_size; ++k) { + auto& vec = to_aggregate[k]; + for (auto idx : vec) { + builder.push_back_elem(var.get(idx)); + break; + } + } + return builder.finish(); +} + +template +std::shared_ptr general_first( + const Var& var, const std::vector>& to_aggregate) { + ValueColumnBuilder builder; + size_t col_size = to_aggregate.size(); + builder.reserve(col_size); + for (size_t k = 0; k < col_size; ++k) { + auto& vec = to_aggregate[k]; + for (auto idx : vec) { + builder.push_back_elem(var.get(idx)); + break; + } + } + return builder.finish(); +} + +template +std::shared_ptr general_min( + const Var& var, const std::vector>& to_aggregate) { + ValueColumnBuilder builder; + size_t col_size = to_aggregate.size(); + builder.reserve(col_size); + for (size_t k = 0; k < col_size; ++k) { + auto& vec = to_aggregate[k]; + if (vec.size() == 0) { + continue; + } + NT s = TypedConverter::to_typed(var.get(vec[0])); + for (auto idx : vec) { + s = std::min(s, TypedConverter::to_typed(var.get(idx))); + } + if constexpr (std::is_same::value) { + builder.push_back_opt(std::string(s)); + } else { + builder.push_back_opt(s); + } + } + return builder.finish(); +} + +template +std::shared_ptr general_max( + const Var& var, const std::vector>& to_aggregate) { + ValueColumnBuilder builder; + size_t col_size = to_aggregate.size(); + builder.reserve(col_size); + for (size_t k = 0; k < col_size; ++k) { + auto& vec = to_aggregate[k]; + if (vec.size() == 0) { + continue; + } + NT s = TypedConverter::to_typed(var.get(vec[0])); + for (auto idx : vec) { + s = std::max(s, TypedConverter::to_typed(var.get(idx))); + } + if constexpr (std::is_same::value) { + builder.push_back_opt(std::string(s)); + } else { + builder.push_back_opt(s); + } + } + return builder.finish(); +} + +std::shared_ptr string_to_set( + const Var& var, const std::vector>& to_aggregate) { + ValueColumnBuilder> builder; + size_t col_size = to_aggregate.size(); + builder.reserve(col_size); + for (size_t k = 0; k < col_size; ++k) { + auto& vec = to_aggregate[k]; + std::set elem; + for (auto idx : vec) { + elem.insert(std::string(var.get(idx).as_string())); + } + builder.push_back_opt(std::move(elem)); + } + return builder.finish(); +} + +std::shared_ptr tuple_to_list( + const Var& var, const std::vector>& to_aggregate) { + ListValueColumnBuilder builder; + size_t col_size = to_aggregate.size(); + builder.reserve(col_size); + std::vector> impls; + for (size_t k = 0; k < col_size; ++k) { + auto& vec = to_aggregate[k]; + + std::vector elem; + for (auto idx : vec) { + elem.push_back(var.get(idx)); + } + auto impl = ListImpl::make_list_impl(elem); + auto list = List::make_list(impl); + impls.emplace_back(impl); + builder.push_back_opt(list); + } + builder.set_list_impls(impls); + return builder.finish(); +} + +std::shared_ptr apply_reduce( + const AggFunc& func, const std::vector>& to_aggregate) { + if (func.aggregate == AggrKind::kSum) { + if (func.vars.size() != 1) { + LOG(FATAL) << "only 1 variable to sum is allowed"; + } + const Var& var = func.vars[0]; + if (var.type() == RTAnyType::kI32Value) { + return numeric_sum(var, to_aggregate); + } else { + LOG(FATAL) << "reduce on " << static_cast(var.type().type_enum_) + << " is not supported..."; + } + } else if (func.aggregate == AggrKind::kToSet) { + if (func.vars.size() != 1) { + LOG(FATAL) << "only 1 variable to to_set is allowed"; + } + const Var& var = func.vars[0]; + if (var.type() == RTAnyType::kStringValue) { + return string_to_set(var, to_aggregate); + } else { + LOG(FATAL) << "not support"; + } + } else if (func.aggregate == AggrKind::kCountDistinct) { + if (func.vars.size() == 1 && func.vars[0].type() == RTAnyType::kVertex) { + const Var& var = func.vars[0]; + return vertex_count_distinct(var, to_aggregate); + } else { + return general_count_distinct(func.vars, to_aggregate); + } + } else if (func.aggregate == AggrKind::kCount) { + // return general_count(to_aggregate); + return general_count(func.vars, to_aggregate); + } else if (func.aggregate == AggrKind::kFirst) { + if (func.vars.size() != 1) { + LOG(FATAL) << "only 1 variable to first is allowed"; + } + + const Var& var = func.vars[0]; + if (var.type() == RTAnyType::kVertex) { + return vertex_first(var, to_aggregate); + } else if (var.type() == RTAnyType::kI64Value) { + return general_first(var, to_aggregate); + } + } else if (func.aggregate == AggrKind::kMin) { + if (func.vars.size() != 1) { + LOG(FATAL) << "only 1 variable to min is allowed"; + } + + const Var& var = func.vars[0]; + if (var.type() == RTAnyType::kI32Value) { + return general_min(var, to_aggregate); + } else if (var.type() == RTAnyType::kStringValue) { + return general_min(var, to_aggregate); + } + } else if (func.aggregate == AggrKind::kMax) { + if (func.vars.size() != 1) { + LOG(FATAL) << "only 1 variable to max is allowed"; + } + + const Var& var = func.vars[0]; + if (var.type() == RTAnyType::kI32Value) { + return general_max(var, to_aggregate); + } else if (var.type() == RTAnyType::kStringValue) { + return general_max(var, to_aggregate); + } + } else if (func.aggregate == AggrKind::kToList) { + const Var& var = func.vars[0]; + if (func.vars.size() != 1) { + LOG(FATAL) << "only 1 variable to to_list is allowed"; + } + if (var.type() == RTAnyType::kTuple) { + return tuple_to_list(var, to_aggregate); + } + } else if (func.aggregate == AggrKind::kAvg) { + if (func.vars.size() != 1) { + LOG(FATAL) << "only 1 variable to avg is allowed"; + } + // LOG(FATAL) << "not support"; + const Var& var = func.vars[0]; + if (var.type() == RTAnyType::kI32Value) { + ValueColumnBuilder builder; + size_t col_size = to_aggregate.size(); + builder.reserve(col_size); + for (size_t k = 0; k < col_size; ++k) { + auto& vec = to_aggregate[k]; + int32_t s = 0; + + for (auto idx : vec) { + s += TypedConverter::to_typed(var.get(idx)); + } + builder.push_back_opt(s / vec.size()); + } + return builder.finish(); + } + } + + LOG(FATAL) << "unsupport " << static_cast(func.aggregate); + return nullptr; +} + +Context eval_group_by(const physical::GroupBy& opr, const ReadTransaction& txn, + Context&& ctx) { + if (ctx.row_num() == 0) { + return ctx; + } + std::vector functions; + std::vector mappings; + int func_num = opr.functions_size(); + for (int i = 0; i < func_num; ++i) { + functions.emplace_back(opr.functions(i), txn, ctx); + } + + int mappings_num = opr.mappings_size(); + // return ctx; + if (mappings_num == 0) { + Context ret; + for (int i = 0; i < func_num; ++i) { + std::vector tmp; + for (size_t _i = 0; _i < ctx.row_num(); ++_i) { + tmp.emplace_back(_i); + } + auto new_col = apply_reduce(functions[i], {tmp}); + ret.set(functions[i].alias, new_col); + ret.append_tag_id(functions[i].alias); + } + + return ret; + } else { + for (int i = 0; i < mappings_num; ++i) { + mappings.emplace_back(opr.mappings(i), txn, ctx); + } + + auto keys_ret = + generate_aggregate_indices(mappings, ctx.row_num(), functions); + std::vector>& to_aggregate = keys_ret.first; + + Context& ret = keys_ret.second; + + // exclude null values + if (func_num == 1 && functions[0].aggregate != AggrKind::kCount && + functions[0].aggregate != AggrKind::kCountDistinct) { + std::vector tmp; + std::vector> tmp_to_aggregate; + for (size_t i = 0; i < to_aggregate.size(); ++i) { + if (to_aggregate[i].size() == 0) { + continue; + } + tmp_to_aggregate.emplace_back(to_aggregate[i]); + tmp.emplace_back(i); + } + ret.reshuffle(tmp); + std::swap(to_aggregate, tmp_to_aggregate); + } + + for (int i = 0; i < func_num; ++i) { + auto new_col = apply_reduce(functions[i], to_aggregate); + ret.set(functions[i].alias, new_col); + ret.append_tag_id(functions[i].alias); + } + return ret; + } +} + +} // namespace runtime + +} // namespace gs \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/adhoc/operators/intersect.cc b/flex/engines/graph_db/runtime/adhoc/operators/intersect.cc new file mode 100644 index 000000000000..083875309f1e --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/operators/intersect.cc @@ -0,0 +1,34 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/common/operators/intersect.h" +#include "flex/engines/graph_db/runtime/adhoc/operators/operators.h" +#include "flex/engines/graph_db/runtime/common/columns/edge_columns.h" + +namespace gs { +namespace runtime { + +Context eval_intersect(const ReadTransaction& txn, + const physical::Intersect& opr, + std::vector&& ctxs) { + int32_t key = opr.key(); + if (ctxs.size() == 1) { + return std::move(ctxs[0]); + } + return Intersect::intersect(std::move(ctxs), key); +} + +} // namespace runtime +} // namespace gs diff --git a/flex/engines/graph_db/runtime/adhoc/operators/join.cc b/flex/engines/graph_db/runtime/adhoc/operators/join.cc new file mode 100644 index 000000000000..96a6a6dea3b6 --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/operators/join.cc @@ -0,0 +1,56 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/common/operators/join.h" +#include "flex/engines/graph_db/runtime/adhoc/operators/operators.h" + +namespace gs { +namespace runtime { +Context eval_join(const physical::Join& opr, Context&& ctx, Context&& ctx2) { + JoinParams p; + auto left_keys = opr.left_keys(); + for (int i = 0; i < left_keys.size(); i++) { + if (!left_keys.Get(i).has_tag()) { + LOG(FATAL) << "left_keys should have tag"; + } + p.left_columns.push_back(left_keys.Get(i).tag().id()); + } + auto right_keys = opr.right_keys(); + for (int i = 0; i < right_keys.size(); i++) { + if (!right_keys.Get(i).has_tag()) { + LOG(FATAL) << "right_keys should have tag"; + } + p.right_columns.push_back(right_keys.Get(i).tag().id()); + } + switch (opr.join_kind()) { + case physical::Join_JoinKind::Join_JoinKind_INNER: + p.join_type = JoinKind::kInnerJoin; + break; + case physical::Join_JoinKind::Join_JoinKind_SEMI: + p.join_type = JoinKind::kSemiJoin; + break; + case physical::Join_JoinKind::Join_JoinKind_ANTI: + p.join_type = JoinKind::kAntiJoin; + break; + case physical::Join_JoinKind::Join_JoinKind_LEFT_OUTER: + p.join_type = JoinKind::kLeftOuterJoin; + break; + default: + LOG(FATAL) << "unsupported join kind" << opr.join_kind(); + } + return Join::join(std::move(ctx), std::move(ctx2), p); +} +} // namespace runtime +} // namespace gs \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/adhoc/operators/limit.cc b/flex/engines/graph_db/runtime/adhoc/operators/limit.cc new file mode 100644 index 000000000000..1b7fc7300e5a --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/operators/limit.cc @@ -0,0 +1,47 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/adhoc/operators/operators.h" +#include "flex/proto_generated_gie/algebra.pb.h" + +namespace gs { +namespace runtime { + +Context eval_limit(const algebra::Limit& opr, Context&& ctx) { + int lower = 0; + int upper = ctx.row_num(); + + if (opr.has_range()) { + lower = std::max(lower, static_cast(opr.range().lower())); + upper = std::min(upper, static_cast(opr.range().upper())); + } + + if (lower >= upper) { + return Context(); + } + if (lower == 0 && static_cast(upper) == ctx.row_num()) { + return std::move(ctx); + } + + std::vector offsets; + for (int i = lower; i < upper; ++i) { + offsets.push_back(i); + } + ctx.reshuffle(offsets); + + return ctx; +} +} // namespace runtime +} // namespace gs diff --git a/flex/engines/graph_db/runtime/adhoc/operators/operators.h b/flex/engines/graph_db/runtime/adhoc/operators/operators.h new file mode 100644 index 000000000000..d72c99ce2111 --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/operators/operators.h @@ -0,0 +1,86 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RUNTIME_ADHOC_OPERATORS_OPERATORS_H_ +#define RUNTIME_ADHOC_OPERATORS_OPERATORS_H_ + +#include "flex/proto_generated_gie/algebra.pb.h" +#include "flex/proto_generated_gie/physical.pb.h" + +#include "flex/engines/graph_db/database/read_transaction.h" +#include "flex/engines/graph_db/runtime/common/context.h" +#include "flex/utils/app_utils.h" + +namespace gs { + +namespace runtime { + +Context eval_dedup(const algebra::Dedup& opr, const ReadTransaction& txn, + Context&& ctx); + +Context eval_group_by(const physical::GroupBy& opr, const ReadTransaction& txn, + Context&& ctx); + +Context eval_order_by(const algebra::OrderBy& opr, const ReadTransaction& txn, + Context&& ctx); + +Context eval_path_expand_v(const physical::PathExpand& opr, + const ReadTransaction& txn, Context&& ctx, + const std::map& params, + const physical::PhysicalOpr_MetaData& meta, + int alias); + +Context eval_path_expand_p(const physical::PathExpand& opr, + const ReadTransaction& txn, Context&& ctx, + const std::map& params, + const physical::PhysicalOpr_MetaData& meta, + int alias); + +Context eval_project(const physical::Project& opr, const ReadTransaction& txn, + Context&& ctx, + const std::map& params, + const std::vector& data_types); + +Context eval_scan(const physical::Scan& scan_opr, const ReadTransaction& txn, + const std::map& params); + +Context eval_select(const algebra::Select& opr, const ReadTransaction& txn, + Context&& ctx, + const std::map& params); + +Context eval_edge_expand(const physical::EdgeExpand& opr, + const ReadTransaction& txn, Context&& ctx, + const std::map& params, + const physical::PhysicalOpr_MetaData& meta); + +Context eval_get_v(const physical::GetV& opr, const ReadTransaction& txn, + Context&& ctx, + const std::map& params); + +Context eval_intersect(const ReadTransaction& txn, + const physical::Intersect& opr, + std::vector&& ctx); + +Context eval_join(const physical::Join& opr, Context&& ctx, Context&& ctx2); + +Context eval_limit(const algebra::Limit& opr, Context&& ctx); + +void eval_sink(const Context& ctx, const ReadTransaction& txn, Encoder& output); + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_ADHOC_OPERATORS_OPERATORS_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/adhoc/operators/order_by.cc b/flex/engines/graph_db/runtime/adhoc/operators/order_by.cc new file mode 100644 index 000000000000..424707c6054c --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/operators/order_by.cc @@ -0,0 +1,93 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include + +#include "flex/engines/graph_db/runtime/adhoc/var.h" +#include "flex/engines/graph_db/runtime/common/operators/order_by.h" + +#include "flex/proto_generated_gie/algebra.pb.h" + +namespace gs { + +namespace runtime { + +class GeneralComparer { + public: + GeneralComparer() : keys_num_(0) {} + ~GeneralComparer() {} + + void add_keys(Var&& key, bool asc) { + keys_.emplace_back(std::move(key)); + order_.push_back(asc); + ++keys_num_; + } + + bool operator()(size_t lhs, size_t rhs) const { + for (size_t k = 0; k < keys_num_; ++k) { + auto& v = keys_[k]; + auto asc = order_[k]; + RTAny lhs_val = v.get(lhs); + RTAny rhs_val = v.get(rhs); + if (lhs_val < rhs_val) { + return asc; + } else if (rhs_val < lhs_val) { + return !asc; + } + } + + return lhs < rhs; + } + + private: + std::vector keys_; + std::vector order_; + size_t keys_num_; +}; + +Context eval_order_by(const algebra::OrderBy& opr, const ReadTransaction& txn, + Context&& ctx) { + if (ctx.row_num() == 0) { + return ctx; + } + int lower = 0; + int upper = std::numeric_limits::max(); + if (opr.has_limit()) { + lower = std::max(lower, static_cast(opr.limit().lower())); + upper = std::min(upper, static_cast(opr.limit().upper())); + } + + GeneralComparer cmp; + int keys_num = opr.pairs_size(); + for (int i = 0; i < keys_num; ++i) { + const algebra::OrderBy_OrderingPair& pair = opr.pairs(i); + Var v(txn, ctx, pair.key(), VarType::kPathVar); + CHECK(pair.order() == algebra::OrderBy_OrderingPair_Order:: + OrderBy_OrderingPair_Order_ASC || + pair.order() == algebra::OrderBy_OrderingPair_Order:: + OrderBy_OrderingPair_Order_DESC); + bool order = + pair.order() == + algebra::OrderBy_OrderingPair_Order::OrderBy_OrderingPair_Order_ASC; + cmp.add_keys(std::move(v), order); + } + + OrderBy::order_by_with_limit(txn, ctx, cmp, lower, upper); + return ctx; +} + +} // namespace runtime + +} // namespace gs \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/adhoc/operators/path_expand.cc b/flex/engines/graph_db/runtime/adhoc/operators/path_expand.cc new file mode 100644 index 000000000000..4341e556b4c9 --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/operators/path_expand.cc @@ -0,0 +1,110 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/common/operators/path_expand.h" +#include "flex/engines/graph_db/runtime/adhoc/operators/operators.h" +#include "flex/engines/graph_db/runtime/adhoc/utils.h" + +namespace gs { + +namespace runtime { + +Context eval_path_expand_v(const physical::PathExpand& opr, + const ReadTransaction& txn, Context&& ctx, + const std::map& params, + const physical::PhysicalOpr_MetaData& meta, + int alias) { + if (ctx.row_num() == 0) { + return ctx; + } + CHECK(opr.has_start_tag()); + int start_tag = opr.start_tag().value(); + CHECK(opr.path_opt() == + physical::PathExpand_PathOpt::PathExpand_PathOpt_ARBITRARY); + if (opr.result_opt() != + physical::PathExpand_ResultOpt::PathExpand_ResultOpt_END_V) { + // LOG(FATAL) << "not support"; + } + CHECK(!opr.is_optional()); + + Direction dir = parse_direction(opr.base().edge_expand().direction()); + CHECK(!opr.base().edge_expand().is_optional()); + const algebra::QueryParams& query_params = opr.base().edge_expand().params(); + PathExpandParams pep; + pep.alias = alias; + pep.dir = dir; + pep.hop_lower = opr.hop_range().lower(); + pep.hop_upper = opr.hop_range().upper(); + for (size_t ci = 0; ci < ctx.col_num(); ++ci) { + if (ctx.get(ci) != nullptr) { + pep.keep_cols.insert(ci); + } + } + pep.start_tag = start_tag; + pep.labels = parse_label_triplets(meta); + if (opr.base().edge_expand().expand_opt() == + physical::EdgeExpand_ExpandOpt::EdgeExpand_ExpandOpt_VERTEX) { + if (query_params.has_predicate()) { + LOG(FATAL) << "not support"; + } else { + return PathExpand::edge_expand_v(txn, std::move(ctx), pep); + } + } else { + LOG(FATAL) << "not support"; + } + + return ctx; +} + +Context eval_path_expand_p(const physical::PathExpand& opr, + const ReadTransaction& txn, Context&& ctx, + const std::map& params, + const physical::PhysicalOpr_MetaData& meta, + int alias) { + CHECK(opr.has_start_tag()); + int start_tag = opr.start_tag().value(); + CHECK(opr.path_opt() == + physical::PathExpand_PathOpt::PathExpand_PathOpt_ARBITRARY); + + CHECK(!opr.is_optional()); + + Direction dir = parse_direction(opr.base().edge_expand().direction()); + CHECK(!opr.base().edge_expand().is_optional()); + const algebra::QueryParams& query_params = opr.base().edge_expand().params(); + PathExpandParams pep; + pep.alias = alias; + pep.dir = dir; + pep.hop_lower = opr.hop_range().lower(); + pep.hop_upper = opr.hop_range().upper(); + for (size_t ci = 0; ci < ctx.col_num(); ++ci) { + if (ctx.get(ci) != nullptr) { + pep.keep_cols.insert(ci); + } + } + pep.start_tag = start_tag; + pep.labels = parse_label_triplets(meta); + + if (query_params.has_predicate()) { + LOG(FATAL) << "not support"; + } else { + return PathExpand::edge_expand_p(txn, std::move(ctx), pep); + } + + return ctx; +} + +} // namespace runtime + +} // namespace gs \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/adhoc/operators/project.cc b/flex/engines/graph_db/runtime/adhoc/operators/project.cc new file mode 100644 index 000000000000..a56d1c52d217 --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/operators/project.cc @@ -0,0 +1,109 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/adhoc/expr.h" +#include "flex/engines/graph_db/runtime/adhoc/operators/operators.h" +#include "flex/engines/graph_db/runtime/adhoc/utils.h" + +namespace gs { + +namespace runtime { + +bool exchange_tag_alias(const physical::Project_ExprAlias& m, int& tag, + int& alias) { + auto expr = m.expr(); + if (expr.operators().size() == 1 && + expr.operators(0).item_case() == common::ExprOpr::kVar) { + auto var = expr.operators(0).var(); + tag = -1; + if (var.has_tag() && !var.has_property()) { + tag = var.tag().id(); + } + alias = -1; + if (m.has_alias()) { + alias = m.alias().value(); + } + if (tag == alias) { + return true; + } + } + return false; +} + +Context eval_project(const physical::Project& opr, const ReadTransaction& txn, + Context&& ctx, + const std::map& params, + const std::vector& data_types) { + if (ctx.row_num() == 0) { + return ctx; + } + bool is_append = opr.is_append(); + Context ret; + if (is_append) { + ret = std::move(ctx); + } + int mappings_size = opr.mappings_size(); + size_t row_num = ctx.row_num(); + std::vector alias_ids; + if (static_cast(mappings_size) == data_types.size()) { + for (int i = 0; i < mappings_size; ++i) { + const physical::Project_ExprAlias& m = opr.mappings(i); + { + int tag, alias; + if (exchange_tag_alias(m, tag, alias)) { + alias_ids.push_back(alias); + ret.set(alias, ctx.get(tag)); + continue; + } + } + Expr expr(txn, ctx, params, m.expr(), VarType::kPathVar); + int alias = -1; + if (m.has_alias()) { + alias = m.alias().value(); + } + alias_ids.push_back(alias); + auto col = build_column(data_types[i], expr, row_num); + ret.set(alias, col); + } + } else { + for (int i = 0; i < mappings_size; ++i) { + const physical::Project_ExprAlias& m = opr.mappings(i); + { + int tag, alias; + if (exchange_tag_alias(m, tag, alias)) { + ret.set(alias, ctx.get(tag)); + alias_ids.push_back(alias); + continue; + } + } + + Expr expr(txn, ctx, params, m.expr(), VarType::kPathVar); + int alias = -1; + if (m.has_alias()) { + alias = m.alias().value(); + } + alias_ids.push_back(alias); + auto col = build_column_beta(expr, row_num); + ret.set(alias, col); + } + } + ret.update_tag_ids(alias_ids); + + return ret; +} + +} // namespace runtime + +} // namespace gs \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/adhoc/operators/scan.cc b/flex/engines/graph_db/runtime/adhoc/operators/scan.cc new file mode 100644 index 000000000000..d37dc6542653 --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/operators/scan.cc @@ -0,0 +1,249 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/common/operators/scan.h" +#include "flex/engines/graph_db/runtime/adhoc/expr_impl.h" +#include "flex/engines/graph_db/runtime/adhoc/operators/operators.h" +namespace gs { + +namespace runtime { + +static bool is_find_vertex(const physical::Scan& scan_opr, + const std::map& params, + label_t& label, int64_t& vertex_id, int& alias, + bool& scan_oid) { + if (scan_opr.scan_opt() != physical::Scan::VERTEX) { + return false; + } + if (scan_opr.has_alias()) { + alias = scan_opr.alias().value(); + } else { + alias = -1; + } + + if (!scan_opr.has_params()) { + return false; + } + const algebra::QueryParams& p = scan_opr.params(); + if (p.tables_size() != 1) { + return false; + } + if (p.has_predicate()) { + return false; + } + const common::NameOrId& table = p.tables(0); + label = static_cast(table.id()); + + if (!scan_opr.has_idx_predicate()) { + return false; + } + const algebra::IndexPredicate& predicate = scan_opr.idx_predicate(); + if (predicate.or_predicates_size() != 1) { + return false; + } + if (predicate.or_predicates(0).predicates_size() != 1) { + return false; + } + const algebra::IndexPredicate_Triplet& triplet = + predicate.or_predicates(0).predicates(0); + if (!triplet.has_key()) { + return false; + } + auto key = triplet.key(); + if (key.has_key()) { + scan_oid = true; + } else if (key.has_id()) { + scan_oid = false; + } else { + LOG(FATAL) << "unexpected key case"; + } + + if (triplet.cmp() != common::Logical::EQ) { + return false; + } + + switch (triplet.value_case()) { + case algebra::IndexPredicate_Triplet::ValueCase::kConst: { + if (triplet.const_().item_case() == common::Value::kI32) { + vertex_id = triplet.const_().i32(); + } else if (triplet.const_().item_case() == common::Value::kI64) { + vertex_id = triplet.const_().i64(); + } else { + LOG(FATAL) << "unexpected value case" << triplet.const_().item_case(); + } + } break; + case algebra::IndexPredicate_Triplet::ValueCase::kParam: { + const common::DynamicParam& p = triplet.param(); + std::string name = p.name(); + std::string value = params.at(name); + vertex_id = std::stoll(value); + } break; + default: { + LOG(FATAL) << "unexpected value case"; + } break; + } + + return true; +} + +bool parse_idx_predicate(const algebra::IndexPredicate& predicate, + const std::map& params, + std::vector& oids, bool& scan_oid) { + if (predicate.or_predicates_size() != 1) { + return false; + } + if (predicate.or_predicates(0).predicates_size() != 1) { + return false; + } + const algebra::IndexPredicate_Triplet& triplet = + predicate.or_predicates(0).predicates(0); + if (!triplet.has_key()) { + return false; + } + auto key = triplet.key(); + if (key.has_key()) { + scan_oid = true; + } else if (key.has_id()) { + scan_oid = false; + } else { + LOG(FATAL) << "unexpected key case"; + } + // const common::Property& key = triplet.key(); + if (triplet.cmp() != common::Logical::EQ && triplet.cmp() != common::WITHIN) { + return false; + } + + if (triplet.value_case() == + algebra::IndexPredicate_Triplet::ValueCase::kConst) { + if (triplet.const_().item_case() == common::Value::kI32) { + oids.emplace_back(triplet.const_().i32()); + } else if (triplet.const_().item_case() == common::Value::kI64) { + oids.emplace_back(triplet.const_().i64()); + } else if (triplet.const_().item_case() == common::Value::kI64Array) { + const auto& arr = triplet.const_().i64_array(); + for (int i = 0; i < arr.item_size(); ++i) { + oids.emplace_back(arr.item(i)); + } + + } else { + LOG(FATAL) << "unexpected value case" << triplet.const_().item_case(); + } + } else if (triplet.value_case() == + algebra::IndexPredicate_Triplet::ValueCase::kParam) { + const common::DynamicParam& p = triplet.param(); + std::string name = p.name(); + std::string value = params.at(name); + oids.emplace_back(std::stoll(value)); + } + return true; +} + +Context eval_scan(const physical::Scan& scan_opr, const ReadTransaction& txn, + const std::map& params) { + label_t label; + int64_t vertex_id; + int alias; + + bool scan_oid; + if (is_find_vertex(scan_opr, params, label, vertex_id, alias, scan_oid)) { + return Scan::find_vertex(txn, label, vertex_id, alias, scan_oid); + } + + const auto& opt = scan_opr.scan_opt(); + if (opt == physical::Scan::VERTEX) { + ScanParams scan_params; + if (scan_opr.has_alias()) { + scan_params.alias = scan_opr.alias().value(); + } else { + scan_params.alias = -1; + } + CHECK(scan_opr.has_params()); + const auto& scan_opr_params = scan_opr.params(); + for (const auto& table : scan_opr_params.tables()) { + scan_params.tables.push_back(table.id()); + } + + if (scan_opr.has_idx_predicate() && scan_opr_params.has_predicate()) { + Context ctx; + auto expr = parse_expression( + txn, ctx, params, scan_opr_params.predicate(), VarType::kVertexVar); + std::vector oids{}; + CHECK(parse_idx_predicate(scan_opr.idx_predicate(), params, oids, + scan_oid)); + if (scan_oid) { + return Scan::scan_vertex( + txn, scan_params, [&expr, &txn, oids](label_t label, vid_t vid) { + return std::find(oids.begin(), oids.end(), + txn.GetVertexId(label, vid).AsInt64()) != + oids.end() && + expr->eval_vertex(label, vid, 0).as_bool(); + }); + } else { + return Scan::scan_gid_vertex( + txn, scan_params, + [&expr, oids](label_t label, vid_t vid) { + return expr->eval_vertex(label, vid, 0).as_bool(); + }, + oids); + } + } + + if (scan_opr.has_idx_predicate()) { + std::vector oids{}; + CHECK(parse_idx_predicate(scan_opr.idx_predicate(), params, oids, + scan_oid)); + + if (scan_oid) { + return Scan::scan_vertex( + txn, scan_params, [&txn, oids](label_t label, vid_t vid) { + return std::find(oids.begin(), oids.end(), + txn.GetVertexId(label, vid).AsInt64()) != + oids.end(); + }); + } else { + return Scan::scan_gid_vertex( + txn, scan_params, [](label_t, vid_t) { return true; }, oids); + } + } + + if (scan_opr_params.has_predicate()) { + Context ctx; + auto expr = parse_expression( + txn, ctx, params, scan_opr_params.predicate(), VarType::kVertexVar); + if (expr->is_optional()) { + return Scan::scan_vertex( + txn, scan_params, [&expr](label_t label, vid_t vid) { + return expr->eval_vertex(label, vid, 0, 0).as_bool(); + }); + } else { + return Scan::scan_vertex( + txn, scan_params, [&expr](label_t label, vid_t vid) { + return expr->eval_vertex(label, vid, 0).as_bool(); + }); + } + } + + if ((!scan_opr.has_idx_predicate()) && (!scan_opr_params.has_predicate())) { + return Scan::scan_vertex(txn, scan_params, + [](label_t, vid_t) { return true; }); + } + } + LOG(FATAL) << "AAAAA"; + return Context(); +} + +} // namespace runtime + +} // namespace gs \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/adhoc/operators/select.cc b/flex/engines/graph_db/runtime/adhoc/operators/select.cc new file mode 100644 index 000000000000..13b78e674ccb --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/operators/select.cc @@ -0,0 +1,51 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/adhoc/expr.h" +#include "flex/engines/graph_db/runtime/adhoc/operators/operators.h" + +namespace gs { + +namespace runtime { + +Context eval_select(const algebra::Select& opr, const ReadTransaction& txn, + Context&& ctx, + const std::map& params) { + Expr expr(txn, ctx, params, opr.predicate(), VarType::kPathVar); + std::vector offsets; + size_t row_num = ctx.row_num(); + if (expr.is_optional()) { + for (size_t i = 0; i < row_num; ++i) { + if (expr.eval_path(i, 0).is_null()) { + continue; + } else if (expr.eval_path(i, 0).as_bool()) { + offsets.push_back(i); + } + } + } else { + for (size_t i = 0; i < row_num; ++i) { + if (expr.eval_path(i).as_bool()) { + offsets.push_back(i); + } + } + } + + ctx.reshuffle(offsets); + return ctx; +} + +} // namespace runtime + +} // namespace gs \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/adhoc/operators/sink.cc b/flex/engines/graph_db/runtime/adhoc/operators/sink.cc new file mode 100644 index 000000000000..ead64bbd39dd --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/operators/sink.cc @@ -0,0 +1,46 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/adhoc/operators/operators.h" +#include "flex/proto_generated_gie/results.pb.h" + +namespace gs { + +namespace runtime { + +void eval_sink(const Context& ctx, const ReadTransaction& txn, + Encoder& output) { + size_t row_num = ctx.row_num(); + results::CollectiveResults results; + for (size_t i = 0; i < row_num; ++i) { + auto result = results.add_results(); + for (size_t j : ctx.tag_ids) { + auto col = ctx.get(j); + if (col == nullptr) { + continue; + } + auto column = result->mutable_record()->add_columns(); + auto val = col->get_elem(i); + val.sink(txn, j, column); + } + } + LOG(INFO) << "sink: " << results.DebugString(); + auto res = results.SerializeAsString(); + output.put_bytes(res.data(), res.size()); +} + +} // namespace runtime + +} // namespace gs \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/adhoc/predicates.h b/flex/engines/graph_db/runtime/adhoc/predicates.h new file mode 100644 index 000000000000..560ffc9c40b7 --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/predicates.h @@ -0,0 +1,88 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RUNTIME_ADHOC_PREDICATES_H_ +#define RUNTIME_ADHOC_PREDICATES_H_ + +#include "flex/engines/graph_db/database/read_transaction.h" +#include "flex/engines/graph_db/runtime/adhoc/expr.h" +#include "flex/engines/graph_db/runtime/adhoc/var.h" +#include "flex/engines/graph_db/runtime/common/context.h" +#include "flex/proto_generated_gie/expr.pb.h" + +namespace gs { + +namespace runtime { + +struct GeneralPathPredicate { + GeneralPathPredicate(const ReadTransaction& txn, const Context& ctx, + const std::map& params, + const common::Expression& expr) + : expr_(txn, ctx, params, expr, VarType::kPathVar) {} + + bool operator()(size_t idx) const { + auto val = expr_.eval_path(idx); + return val.as_bool(); + } + + Expr expr_; +}; + +struct GeneralVertexPredicate { + GeneralVertexPredicate(const ReadTransaction& txn, const Context& ctx, + const std::map& params, + const common::Expression& expr) + : expr_(txn, ctx, params, expr, VarType::kVertexVar) {} + + bool operator()(label_t label, vid_t v, size_t path_idx) const { + auto val = expr_.eval_vertex(label, v, path_idx); + return val.as_bool(); + } + + Expr expr_; +}; + +struct GeneralEdgePredicate { + GeneralEdgePredicate(const ReadTransaction& txn, const Context& ctx, + const std::map& params, + const common::Expression& expr) + : expr_(txn, ctx, params, expr, VarType::kEdgeVar) {} + + bool operator()(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& edata, Direction dir, size_t path_idx) const { + auto val = expr_.eval_edge(label, src, dst, edata, path_idx); + return val.as_bool(); + } + + Expr expr_; +}; + +struct DummyVertexPredicate { + bool operator()(label_t label, vid_t v, size_t path_idx) const { + return true; + } +}; + +struct DummyEdgePredicate { + bool operator()(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& edata, Direction dir, size_t path_idx) const { + return true; + } +}; + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_ADHOC_PREDICATES_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/adhoc/runtime.cc b/flex/engines/graph_db/runtime/adhoc/runtime.cc new file mode 100644 index 000000000000..2405dc805cbc --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/runtime.cc @@ -0,0 +1,236 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/adhoc/runtime.h" + +namespace gs { + +namespace runtime { + +class OpCost { + public: + OpCost() {} + ~OpCost() { + double total = 0; + for (auto& pair : table) { + total += pair.second; + } + LOG(INFO) << "op elapsed time: "; + for (auto& pair : table) { + LOG(INFO) << "\t" << pair.first << ": " << pair.second << " (" + << pair.second / total * 100.0 << "%)"; + } + } + + static OpCost& get() { + static OpCost instance; + return instance; + } + + std::map table; +}; + +static std::string get_opr_name(const physical::PhysicalOpr& opr) { + switch (opr.opr().op_kind_case()) { + case physical::PhysicalOpr_Operator::OpKindCase::kScan: { + return "scan"; + } + case physical::PhysicalOpr_Operator::OpKindCase::kEdge: { + return "edge_expand"; + } + case physical::PhysicalOpr_Operator::OpKindCase::kVertex: { + return "get_v"; + } + case physical::PhysicalOpr_Operator::OpKindCase::kOrderBy: { + return "order_by"; + } + case physical::PhysicalOpr_Operator::OpKindCase::kProject: { + return "project"; + } + case physical::PhysicalOpr_Operator::OpKindCase::kSink: { + return "sink"; + } + case physical::PhysicalOpr_Operator::OpKindCase::kDedup: { + return "dedup"; + } + case physical::PhysicalOpr_Operator::OpKindCase::kGroupBy: { + return "group_by"; + } + case physical::PhysicalOpr_Operator::OpKindCase::kSelect: { + return "select"; + } + case physical::PhysicalOpr_Operator::OpKindCase::kPath: { + return "path"; + } + default: + return "unknown - " + + std::to_string(static_cast(opr.opr().op_kind_case())); + } +} + +Context runtime_eval_impl(const physical::PhysicalPlan& plan, Context&& ctx, + const ReadTransaction& txn, + const std::map& params) { + Context ret = ctx; + + auto& op_cost = OpCost::get().table; + + int opr_num = plan.plan_size(); + bool terminate = false; + for (int i = 0; i < opr_num; ++i) { + const physical::PhysicalOpr& opr = plan.plan(i); + double t = -grape::GetCurrentTime(); + assert(opr.has_opr()); + switch (opr.opr().op_kind_case()) { + LOG(INFO) << "eval: " << get_opr_name(opr); + case physical::PhysicalOpr_Operator::OpKindCase::kScan: { + ret = eval_scan(opr.opr().scan(), txn, params); + t += grape::GetCurrentTime(); + op_cost["scan"] += t; + } break; + case physical::PhysicalOpr_Operator::OpKindCase::kEdge: { + CHECK_EQ(opr.meta_data_size(), 1); + ret = eval_edge_expand(opr.opr().edge(), txn, std::move(ret), params, + opr.meta_data(0)); + t += grape::GetCurrentTime(); + op_cost["edge_expand"] += t; + } break; + case physical::PhysicalOpr_Operator::OpKindCase::kVertex: { + ret = eval_get_v(opr.opr().vertex(), txn, std::move(ret), params); + t += grape::GetCurrentTime(); + op_cost["get_v"] += t; + } break; + case physical::PhysicalOpr_Operator::OpKindCase::kProject: { + std::vector data_types; + if (opr.meta_data_size() == opr.opr().project().mappings_size()) { + for (int i = 0; i < opr.meta_data_size(); ++i) { + if (opr.meta_data(i).type().type_case() == + common::IrDataType::TypeCase::TYPE_NOT_SET) { + LOG(INFO) << "type not set"; + } + data_types.push_back(opr.meta_data(i).type()); + } + } + ret = eval_project(opr.opr().project(), txn, std::move(ret), params, + data_types); + t += grape::GetCurrentTime(); + op_cost["project"] += t; + } break; + case physical::PhysicalOpr_Operator::OpKindCase::kOrderBy: { + ret = eval_order_by(opr.opr().order_by(), txn, std::move(ret)); + t += grape::GetCurrentTime(); + op_cost["order_by"] += t; + } break; + case physical::PhysicalOpr_Operator::OpKindCase::kGroupBy: { + ret = eval_group_by(opr.opr().group_by(), txn, std::move(ret)); + t += grape::GetCurrentTime(); + op_cost["group_by"] += t; + } break; + case physical::PhysicalOpr_Operator::OpKindCase::kDedup: { + ret = eval_dedup(opr.opr().dedup(), txn, std::move(ret)); + t += grape::GetCurrentTime(); + op_cost["dedup"] += t; + } break; + case physical::PhysicalOpr_Operator::OpKindCase::kSelect: { + ret = eval_select(opr.opr().select(), txn, std::move(ret), params); + t += grape::GetCurrentTime(); + op_cost["select"] += t; + } break; + case physical::PhysicalOpr_Operator::OpKindCase::kPath: { + if ((i + 1) < opr_num) { + const physical::PhysicalOpr& next_opr = plan.plan(i + 1); + if (next_opr.opr().has_vertex() && + opr.opr().path().result_opt() == + physical::PathExpand_ResultOpt::PathExpand_ResultOpt_END_V && + opr.opr().path().base().edge_expand().expand_opt() == + physical::EdgeExpand_ExpandOpt::EdgeExpand_ExpandOpt_VERTEX) { + int alias = -1; + if (next_opr.opr().vertex().has_alias()) { + alias = next_opr.opr().vertex().alias().value(); + } + ret = eval_path_expand_v(opr.opr().path(), txn, std::move(ret), + params, opr.meta_data(0), alias); + ++i; + } else { + int alias = -1; + if (opr.opr().path().has_alias()) { + alias = opr.opr().path().alias().value(); + } + ret = eval_path_expand_p(opr.opr().path(), txn, std::move(ret), + params, opr.meta_data(0), alias); + } + } else { + LOG(FATAL) << "not support"; + } + t += grape::GetCurrentTime(); + op_cost["path_expand"] += t; + } break; + case physical::PhysicalOpr_Operator::OpKindCase::kSink: { + terminate = true; + } break; + case physical::PhysicalOpr_Operator::OpKindCase::kRoot: { + // do nothing + } break; + case physical::PhysicalOpr_Operator::OpKindCase::kJoin: { + auto op = opr.opr().join(); + auto ret_dup = ret.dup(); + auto ctx = runtime_eval_impl(op.left_plan(), std::move(ret), txn, params); + auto ctx2 = + runtime_eval_impl(op.right_plan(), std::move(ret_dup), txn, params); + ret = eval_join(op, std::move(ctx), std::move(ctx2)); + + } break; + case physical::PhysicalOpr_Operator::OpKindCase::kIntersect: { + auto op = opr.opr().intersect(); + size_t num = op.sub_plans_size(); + std::vector ctxs; + ret.push_idx_col(); + for (size_t i = 0; i < num; ++i) { + if (i + 1 < num) { + auto ret_dup = ret.dup(); + ctxs.push_back(runtime_eval_impl(op.sub_plans(i), std::move(ret_dup), + txn, params)); + } else { + ctxs.push_back( + runtime_eval_impl(op.sub_plans(i), std::move(ret), txn, params)); + } + } + ret = eval_intersect(txn, op, std::move(ctxs)); + } break; + case physical::PhysicalOpr_Operator::OpKindCase::kLimit: { + ret = eval_limit(opr.opr().limit(), std::move(ret)); + } break; + + default: + LOG(FATAL) << "opr not support..." << get_opr_name(opr) + << opr.DebugString(); + break; + } + if (terminate) { + break; + } + } + return ret; +} + +Context runtime_eval(const physical::PhysicalPlan& plan, + const ReadTransaction& txn, + const std::map& params) { + return runtime_eval_impl(plan, Context(), txn, params); +} + +} // namespace runtime + +} // namespace gs \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/adhoc/runtime.h b/flex/engines/graph_db/runtime/adhoc/runtime.h new file mode 100644 index 000000000000..9d9ccf5fd1be --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/runtime.h @@ -0,0 +1,33 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RUNTIME_ADHOC_RUNTIME_H_ +#define RUNTIME_ADHOC_RUNTIME_H_ + +#include "flex/engines/graph_db/runtime/adhoc/operators/operators.h" +#include "flex/proto_generated_gie/physical.pb.h" + +namespace gs { + +namespace runtime { + +Context runtime_eval(const physical::PhysicalPlan& plan, + const ReadTransaction& txn, + const std::map& params); + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_ADHOC_RUNTIME_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/adhoc/utils.cc b/flex/engines/graph_db/runtime/adhoc/utils.cc new file mode 100644 index 000000000000..14799f55d556 --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/utils.cc @@ -0,0 +1,550 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include "flex/engines/graph_db/runtime/adhoc/utils.h" +#include "flex/engines/graph_db/runtime/common/columns/value_columns.h" +#include "flex/engines/graph_db/runtime/common/columns/vertex_columns.h" + +namespace gs { + +namespace runtime { + +Direction parse_direction(const physical::EdgeExpand_Direction& dir) { + if (dir == physical::EdgeExpand_Direction_OUT) { + return Direction::kOut; + } else if (dir == physical::EdgeExpand_Direction_IN) { + return Direction::kIn; + } else if (dir == physical::EdgeExpand_Direction_BOTH) { + return Direction::kBoth; + } + LOG(FATAL) << "not support..."; + return Direction::kOut; +} + +std::vector parse_tables(const algebra::QueryParams& query_params) { + std::vector tables; + int tn = query_params.tables_size(); + for (int i = 0; i < tn; ++i) { + const common::NameOrId& table = query_params.tables(i); + tables.push_back(static_cast(table.id())); + } + return tables; +} + +std::vector parse_label_triplets( + const physical::PhysicalOpr_MetaData& meta) { + std::vector labels; + if (meta.has_type()) { + const common::IrDataType& t = meta.type(); + if (t.has_graph_type()) { + const common::GraphDataType& gt = t.graph_type(); + if (gt.element_opt() == common::GraphDataType_GraphElementOpt:: + GraphDataType_GraphElementOpt_EDGE) { + int label_num = gt.graph_data_type_size(); + for (int label_i = 0; label_i < label_num; ++label_i) { + const common::GraphDataType_GraphElementLabel& gdt = + gt.graph_data_type(label_i).label(); + labels.emplace_back(static_cast(gdt.src_label().value()), + static_cast(gdt.dst_label().value()), + static_cast(gdt.label())); + } + } + } + } + return labels; +} + +std::shared_ptr create_column( + const common::IrDataType& data_type) { + switch (data_type.type_case()) { + case common::IrDataType::kDataType: + LOG(FATAL) << "not support"; + break; + case common::IrDataType::kGraphType: { + const common::GraphDataType& graph_data_type = data_type.graph_type(); + common::GraphDataType_GraphElementOpt elem_opt = + graph_data_type.element_opt(); + int label_num = graph_data_type.graph_data_type_size(); + if (elem_opt == common::GraphDataType_GraphElementOpt:: + GraphDataType_GraphElementOpt_VERTEX) { + if (label_num == 1) { + label_t v_label = static_cast( + graph_data_type.graph_data_type(0).label().label()); + return std::make_shared(v_label); + } else if (label_num > 1) { + return std::make_shared(); + } else { + LOG(FATAL) << "unexpected type"; + } + } else if (elem_opt == common::GraphDataType_GraphElementOpt:: + GraphDataType_GraphElementOpt_EDGE) { + LOG(FATAL) << "unexpected type"; + } else { + LOG(FATAL) << "unexpected type"; + } + } break; + default: + LOG(FATAL) << "unexpected type"; + break; + } + return nullptr; +} + +std::shared_ptr create_column_beta(RTAnyType type) { + switch (type.type_enum_) { + case RTAnyType::RTAnyTypeImpl::kI64Value: + return std::make_shared>(); + case RTAnyType::RTAnyTypeImpl::kStringValue: + return std::make_shared>(); + case RTAnyType::RTAnyTypeImpl::kVertex: + return std::make_shared(); + default: + LOG(FATAL) << "unsupport type: " << static_cast(type.type_enum_); + break; + } + return nullptr; +} + +std::shared_ptr create_column_builder(RTAnyType type) { + switch (type.type_enum_) { + case RTAnyType::RTAnyTypeImpl::kI64Value: + return std::make_shared>(); + case RTAnyType::RTAnyTypeImpl::kStringValue: + return std::make_shared>(); + case RTAnyType::RTAnyTypeImpl::kVertex: + return std::make_shared(); + case RTAnyType::RTAnyTypeImpl::kI32Value: + return std::make_shared>(); + case RTAnyType::RTAnyTypeImpl::kDate32: + return std::make_shared>(); + case RTAnyType::RTAnyTypeImpl::kU64Value: + return std::make_shared>(); + case RTAnyType::RTAnyTypeImpl::kBoolValue: + // fix me + return std::make_shared>(); + default: + LOG(FATAL) << "unsupport type: " << static_cast(type.type_enum_); + break; + } + return nullptr; +} + +std::shared_ptr build_optional_column( + const common::IrDataType& data_type, const Expr& expr, size_t row_num) { + switch (data_type.type_case()) { + case common::IrDataType::kDataType: { + switch (data_type.data_type()) { + case common::DataType::INT64: { + OptionalValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i, 0); + if (v.is_null()) { + builder.push_back_null(); + } else { + builder.push_back_opt(v.as_int64(), true); + } + } + + return builder.finish(); + } break; + case common::DataType::INT32: { + OptionalValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i, 0); + if (v.is_null()) { + builder.push_back_null(); + } else { + builder.push_back_opt(v.as_int32(), true); + } + } + + return builder.finish(); + } break; + case common::DataType::DOUBLE: { + OptionalValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i, 0); + if (v.is_null()) { + builder.push_back_null(); + } else { + builder.push_back_opt(v.as_double(), true); + } + } + + return builder.finish(); + } break; + case common::DataType::BOOLEAN: { + OptionalValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i, 0); + if (v.is_null()) { + builder.push_back_null(); + } else { + builder.push_back_opt(v.as_bool(), true); + } + } + + return builder.finish(); + } break; + case common::DataType::STRING: { + OptionalValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i, 0); + if (v.is_null()) { + builder.push_back_null(); + } else { + builder.push_back_opt(std::string(v.as_string()), true); + } + } + + return builder.finish(); + } break; + + default: { + LOG(FATAL) << "not support"; + break; + } + } + } + case common::IrDataType::TYPE_NOT_SET: { + return build_column_beta(expr, row_num); + } break; + default: { + LOG(FATAL) << "not support" << data_type.DebugString(); + break; + } + } + return nullptr; +} + +std::shared_ptr build_column( + const common::IrDataType& data_type, const Expr& expr, size_t row_num) { + if (expr.is_optional()) { + return build_optional_column(data_type, expr, row_num); + } + switch (data_type.type_case()) { + case common::IrDataType::kDataType: { + switch (data_type.data_type()) { + case common::DataType::INT64: { + ValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i).as_int64(); + builder.push_back_opt(v); + } + + return builder.finish(); + } break; + case common::DataType::INT32: { + ValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i).as_int32(); + builder.push_back_opt(v); + } + + return builder.finish(); + } break; + case common::DataType::STRING: { + ValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i).as_string(); + builder.push_back_opt(std::string(v)); + } + + return builder.finish(); + } break; + case common::DataType::DATE32: { + ValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i).as_date32(); + builder.push_back_opt(v); + } + + return builder.finish(); + } break; + case common::DataType::STRING_ARRAY: { + ValueColumnBuilder> builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + const auto& v = expr.eval_path(i).as_string_set(); + builder.push_back_opt(v); + } + + return builder.finish(); + } break; + case common::DataType::TIMESTAMP: { + ValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i).as_date32(); + builder.push_back_opt(v); + } + + return builder.finish(); + } break; + case common::DataType::BOOLEAN: { + ValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i).as_bool(); + builder.push_back_opt(v); + } + return builder.finish(); + } break; + case common::DataType::DOUBLE: { + ValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i).as_double(); + builder.push_back_opt(v); + } + return builder.finish(); + } break; + default: { + LOG(FATAL) << "not support: " + << common::DataType_Name(data_type.data_type()); + } + } + } break; + case common::IrDataType::kGraphType: { + const common::GraphDataType& graph_data_type = data_type.graph_type(); + common::GraphDataType_GraphElementOpt elem_opt = + graph_data_type.element_opt(); + int label_num = graph_data_type.graph_data_type_size(); + if (elem_opt == common::GraphDataType_GraphElementOpt:: + GraphDataType_GraphElementOpt_VERTEX) { + if (label_num == 1) { + label_t v_label = static_cast( + graph_data_type.graph_data_type(0).label().label()); + SLVertexColumnBuilder builder(v_label); + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + builder.push_back_opt(expr.eval_path(i).as_vertex().second); + } + + return builder.finish(); + } else if (label_num > 1) { + MLVertexColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + builder.push_back_vertex(expr.eval_path(i).as_vertex()); + } + + return builder.finish(); + } else { + LOG(FATAL) << "unexpected type"; + } + } else if (elem_opt == common::GraphDataType_GraphElementOpt:: + GraphDataType_GraphElementOpt_EDGE) { + // LOG(FATAL) << "unexpected type"; + BDMLEdgeColumnBuilder builder; + for (size_t i = 0; i < row_num; ++i) { + builder.push_back_elem(expr.eval_path(i)); + } + return builder.finish(); + } else { + LOG(FATAL) << "unexpected type"; + } + } break; + case common::IrDataType::TYPE_NOT_SET: { + return build_column_beta(expr, row_num); + } break; + default: + LOG(FATAL) << "unexpected type" + << common::DataType_Name(data_type.data_type()); + break; + } + + return nullptr; +} + +std::shared_ptr build_optional_column_beta(const Expr& expr, + size_t row_num) { + switch (expr.type().type_enum_) { + case RTAnyType::RTAnyTypeImpl::kI64Value: { + OptionalValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i, 0); + if (v.is_null()) { + builder.push_back_null(); + } else { + builder.push_back_opt(v.as_int64(), true); + } + } + + return builder.finish(); + } break; + case RTAnyType::RTAnyTypeImpl::kI32Value: { + OptionalValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i, 0); + if (v.is_null()) { + builder.push_back_null(); + } else { + builder.push_back_opt(v.as_int32(), true); + } + } + + return builder.finish(); + } break; + case RTAnyType::RTAnyTypeImpl::kF64Value: { + OptionalValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i, 0); + if (v.is_null()) { + builder.push_back_null(); + } else { + builder.push_back_opt(v.as_double(), true); + } + } + + return builder.finish(); + } break; + case RTAnyType::RTAnyTypeImpl::kMap: { + auto builder = expr.builder(); + for (size_t i = 0; i < row_num; ++i) { + builder->push_back_elem(expr.eval_path(i, 0)); + } + return builder->finish(); + } break; + default: { + LOG(FATAL) << "not support"; + break; + } + } + return nullptr; +} + +std::shared_ptr build_column_beta(const Expr& expr, + size_t row_num) { + if (expr.is_optional()) { + return build_optional_column_beta(expr, row_num); + } + switch (expr.type().type_enum_) { + case RTAnyType::RTAnyTypeImpl::kI64Value: { + ValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + builder.push_back_opt(expr.eval_path(i).as_int64()); + } + + return builder.finish(); + } break; + case RTAnyType::RTAnyTypeImpl::kStringValue: { + ValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + builder.push_back_opt(std::string(expr.eval_path(i).as_string())); + } + + return builder.finish(); + } break; + case RTAnyType::RTAnyTypeImpl::kDate32: { + ValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + builder.push_back_opt(expr.eval_path(i).as_date32()); + } + + return builder.finish(); + } break; + case RTAnyType::RTAnyTypeImpl::kVertex: { + MLVertexColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + builder.push_back_vertex(expr.eval_path(i).as_vertex()); + } + + return builder.finish(); + } break; + case RTAnyType::RTAnyTypeImpl::kI32Value: { + ValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + builder.push_back_opt(expr.eval_path(i).as_int32()); + } + + return builder.finish(); + } break; + case RTAnyType::RTAnyTypeImpl::kF64Value: { + ValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t i = 0; i < row_num; ++i) { + builder.push_back_opt(expr.eval_path(i).as_double()); + } + return builder.finish(); + } break; + case RTAnyType::RTAnyTypeImpl::kEdge: { + BDMLEdgeColumnBuilder builder; + for (size_t i = 0; i < row_num; ++i) { + builder.push_back_elem(expr.eval_path(i)); + } + return builder.finish(); + } + case RTAnyType::RTAnyTypeImpl::kTuple: { + if (expr.type().null_able_) { + OptionalValueColumnBuilder builder; + for (size_t i = 0; i < row_num; ++i) { + auto v = expr.eval_path(i); + if (v.is_null()) { + builder.push_back_null(); + } else { + builder.push_back_elem(v); + } + } + return builder.finish(); + } else { + ValueColumnBuilder builder; + for (size_t i = 0; i < row_num; ++i) { + builder.push_back_elem(expr.eval_path(i)); + } + return builder.finish(); + } + } + case RTAnyType::RTAnyTypeImpl::kList: { + auto builder = expr.builder(); + for (size_t i = 0; i < row_num; ++i) { + builder->push_back_elem(expr.eval_path(i)); + } + return builder->finish(); + } + case RTAnyType::RTAnyTypeImpl::kMap: { + auto builder = expr.builder(); + for (size_t i = 0; i < row_num; ++i) { + builder->push_back_elem(expr.eval_path(i)); + } + return builder->finish(); + } + default: + LOG(FATAL) << "not support - " << static_cast(expr.type().type_enum_); + break; + } + + return nullptr; +} + +} // namespace runtime + +} // namespace gs diff --git a/flex/engines/graph_db/runtime/adhoc/utils.h b/flex/engines/graph_db/runtime/adhoc/utils.h new file mode 100644 index 000000000000..41ed732c9af5 --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/utils.h @@ -0,0 +1,56 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RUNTIME_ADHOC_UTILS_H_ +#define RUNTIME_ADHOC_UTILS_H_ + +#include "flex/engines/graph_db/runtime/adhoc/expr.h" +#include "flex/engines/graph_db/runtime/common/columns/i_context_column.h" +#include "flex/engines/graph_db/runtime/common/rt_any.h" +#include "flex/engines/graph_db/runtime/common/types.h" + +#include "flex/proto_generated_gie/algebra.pb.h" +#include "flex/proto_generated_gie/physical.pb.h" +#include "flex/proto_generated_gie/type.pb.h" + +namespace gs { + +namespace runtime { + +Direction parse_direction(const physical::EdgeExpand_Direction& dir); + +std::vector parse_tables(const algebra::QueryParams& query_params); + +std::vector parse_label_triplets( + const physical::PhysicalOpr_MetaData& meta); + +std::shared_ptr create_column( + const common::IrDataType& data_type); + +std::shared_ptr create_column_beta(RTAnyType type); + +std::shared_ptr build_column( + const common::IrDataType& data_type, const Expr& expr, size_t row_num); + +std::shared_ptr build_column_beta(const Expr& expr, + size_t row_num); + +std::shared_ptr create_column_builder(RTAnyType type); + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_ADHOC_UTILS_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/adhoc/var.cc b/flex/engines/graph_db/runtime/adhoc/var.cc new file mode 100644 index 000000000000..fa15f61a4f41 --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/var.cc @@ -0,0 +1,175 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/adhoc/var.h" +#include "flex/engines/graph_db/runtime/common/accessors.h" +#include "flex/engines/graph_db/runtime/common/columns/edge_columns.h" +#include "flex/engines/graph_db/runtime/common/columns/i_context_column.h" +#include "flex/engines/graph_db/runtime/common/columns/vertex_columns.h" + +namespace gs { + +namespace runtime { + +Var::Var(const ReadTransaction& txn, const Context& ctx, + const common::Variable& pb, VarType var_type) + : getter_(nullptr) { + int tag = -1; + type_ = RTAnyType::kUnknown; + if (pb.has_node_type()) { + type_ = parse_from_ir_data_type(pb.node_type()); + } + if (pb.has_tag()) { + tag = pb.tag().id(); + } + + if (type_ == RTAnyType::kUnknown) { + if (pb.has_tag()) { + tag = pb.tag().id(); + CHECK(ctx.get(tag) != nullptr); + type_ = ctx.get(tag)->elem_type(); + } else if (pb.has_property() && pb.property().has_label()) { + type_ = RTAnyType::kI64Value; + } else { + LOG(FATAL) << "not support"; + } + } + + if (pb.has_tag() || var_type == VarType::kPathVar) { + if (ctx.get(tag)->column_type() == ContextColumnType::kVertex) { + if (pb.has_property()) { + auto& pt = pb.property(); + if (pt.has_id()) { + getter_ = std::make_shared(ctx, tag); + } else if (pt.has_key()) { + if (pt.key().name() == "id") { + getter_ = std::make_shared(txn, ctx, tag); + } else { + getter_ = create_vertex_property_path_accessor(txn, ctx, tag, type_, + pt.key().name()); + } + } else if (pt.has_label()) { + getter_ = create_vertex_label_path_accessor(ctx, tag); + } else { + LOG(FATAL) << "xxx, " << pt.item_case(); + } + } else { + getter_ = std::make_shared(ctx, tag); + } + } else if (ctx.get(tag)->column_type() == ContextColumnType::kValue || + ctx.get(tag)->column_type() == + ContextColumnType::kOptionalValue) { + getter_ = create_context_value_accessor(ctx, tag, type_); + } else if (ctx.get(tag)->column_type() == ContextColumnType::kEdge) { + if (pb.has_property()) { + auto& pt = pb.property(); + if (pt.has_key()) { + getter_ = create_edge_property_path_accessor(ctx, tag, type_); + } else if (pt.has_label()) { + getter_ = create_edge_label_path_accessor(ctx, tag); + } else { + LOG(FATAL) << "not support..."; + } + } else { + getter_ = std::make_shared(ctx, tag); + // LOG(FATAL) << "not support for edge column - " << tag; + } + } else if (ctx.get(tag)->column_type() == ContextColumnType::kPath) { + if (pb.has_property()) { + auto& pt = pb.property(); + if (pt.has_len()) { + getter_ = std::make_shared(ctx, tag); + } else { + LOG(FATAL) << "not support for path column - " << pt.DebugString(); + } + } else { + getter_ = std::make_shared(ctx, tag); + } + } else { + LOG(FATAL) << "not support for " << ctx.get(tag)->column_info(); + } + } else { + if (var_type == VarType::kVertexVar) { + if (pb.has_property()) { + auto& pt = pb.property(); + if (pt.has_id()) { + getter_ = std::make_shared(); + } else if (pt.has_key()) { + if (pt.key().name() == "id") { + getter_ = std::make_shared(txn); + } else { + getter_ = create_vertex_property_vertex_accessor(txn, type_, + pt.key().name()); + } + } else if (pt.has_label()) { + getter_ = std::make_shared(); + } else { + LOG(FATAL) << "xxx, " << pt.item_case(); + } + } else { + LOG(FATAL) << "not support"; + } + } else if (var_type == VarType::kEdgeVar) { + if (pb.has_property()) { + auto& pt = pb.property(); + if (pt.has_key()) { + getter_ = create_edge_property_edge_accessor(type_); + } else { + LOG(FATAL) << "not support"; + } + } else { + LOG(FATAL) << "not support"; + } + } else { + LOG(FATAL) << "not support"; + } + } +} + +Var::~Var() {} + +RTAny Var::get(size_t path_idx) const { return getter_->eval_path(path_idx); } + +RTAny Var::get(size_t path_idx, int) const { + return getter_->eval_path(path_idx, 0); +} + +RTAny Var::get_vertex(label_t label, vid_t v, size_t idx) const { + return getter_->eval_vertex(label, v, idx); +} + +RTAny Var::get_vertex(label_t label, vid_t v, size_t idx, int) const { + return getter_->eval_vertex(label, v, idx, 0); +} + +RTAny Var::get_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const { + return getter_->eval_edge(label, src, dst, data, idx); +} + +RTAny Var::get_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx, int) const { + return getter_->eval_edge(label, src, dst, data, idx, 0); +} + +RTAnyType Var::type() const { return type_; } + +std::shared_ptr Var::builder() const { + return getter_->builder(); +} + +} // namespace runtime + +} // namespace gs diff --git a/flex/engines/graph_db/runtime/adhoc/var.h b/flex/engines/graph_db/runtime/adhoc/var.h new file mode 100644 index 000000000000..a209e49b4a24 --- /dev/null +++ b/flex/engines/graph_db/runtime/adhoc/var.h @@ -0,0 +1,74 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RUNTIME_ADHOC_VAR_H_ +#define RUNTIME_ADHOC_VAR_H_ + +#include "flex/engines/graph_db/database/read_transaction.h" + +#include "flex/engines/graph_db/runtime/common/accessors.h" +#include "flex/engines/graph_db/runtime/common/context.h" + +#include "flex/proto_generated_gie/expr.pb.h" + +namespace gs { + +namespace runtime { + +enum class VarType { + kVertexVar, + kEdgeVar, + kPathVar, +}; + +class VarGetterBase { + public: + virtual ~VarGetterBase() = default; + virtual RTAny eval_path(size_t idx) const = 0; + virtual RTAny eval_vertex(label_t label, vid_t v, size_t idx) const = 0; + virtual RTAny eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const = 0; + virtual std::string name() const = 0; +}; + +class Var { + public: + Var(const ReadTransaction& txn, const Context& ctx, + const common::Variable& pb, VarType var_type); + ~Var(); + + RTAny get(size_t path_idx) const; + RTAny get_vertex(label_t label, vid_t v, size_t idx) const; + RTAny get_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const; + + RTAny get(size_t path_idx, int) const; + RTAny get_vertex(label_t label, vid_t v, size_t idx, int) const; + RTAny get_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx, int) const; + RTAnyType type() const; + bool is_optional() const { return getter_->is_optional(); } + std::shared_ptr builder() const; + + private: + std::shared_ptr getter_; + RTAnyType type_; +}; + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_ADHOC_VAR_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/accessors.cc b/flex/engines/graph_db/runtime/common/accessors.cc new file mode 100644 index 000000000000..de6f19986a6f --- /dev/null +++ b/flex/engines/graph_db/runtime/common/accessors.cc @@ -0,0 +1,155 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/common/accessors.h" +#include "flex/engines/graph_db/runtime/common/rt_any.h" + +namespace gs { + +namespace runtime { + +std::shared_ptr create_context_value_accessor(const Context& ctx, + int tag, + RTAnyType type) { + auto col = ctx.get(tag); + switch (type.type_enum_) { + case RTAnyType::RTAnyTypeImpl::kI64Value: + return std::make_shared>(ctx, tag); + case RTAnyType::RTAnyTypeImpl::kI32Value: + return std::make_shared>(ctx, tag); + case RTAnyType::RTAnyTypeImpl::kU64Value: + return std::make_shared>(ctx, tag); + case RTAnyType::RTAnyTypeImpl::kStringValue: + return std::make_shared>(ctx, tag); + case RTAnyType::RTAnyTypeImpl::kDate32: + return std::make_shared>(ctx, tag); + case RTAnyType::RTAnyTypeImpl::kStringSetValue: + return std::make_shared>>(ctx, + tag); + case RTAnyType::RTAnyTypeImpl::kBoolValue: + return std::make_shared>(ctx, tag); + case RTAnyType::RTAnyTypeImpl::kTuple: + return std::make_shared>(ctx, tag); + case RTAnyType::RTAnyTypeImpl::kList: + return std::make_shared>(ctx, tag); + + default: + LOG(FATAL) << "not implemented - " << static_cast(type.type_enum_); + } + return nullptr; +} + +std::shared_ptr create_vertex_property_path_accessor( + const ReadTransaction& txn, const Context& ctx, int tag, RTAnyType type, + const std::string& prop_name) { + switch (type.type_enum_) { + case RTAnyType::RTAnyTypeImpl::kI64Value: + return std::make_shared>(txn, ctx, tag, + prop_name); + case RTAnyType::RTAnyTypeImpl::kI32Value: + return std::make_shared>(txn, ctx, tag, + prop_name); + case RTAnyType::RTAnyTypeImpl::kU64Value: + return std::make_shared>(txn, ctx, tag, + prop_name); + case RTAnyType::RTAnyTypeImpl::kStringValue: + return std::make_shared>( + txn, ctx, tag, prop_name); + case RTAnyType::RTAnyTypeImpl::kDate32: + return std::make_shared>(txn, ctx, tag, + prop_name); + default: + LOG(FATAL) << "not implemented - " << static_cast(type.type_enum_); + } + return nullptr; +} + +std::shared_ptr create_vertex_label_path_accessor(const Context& ctx, + int tag) { + return std::make_shared(ctx, tag); +} + +std::shared_ptr create_vertex_property_vertex_accessor( + const ReadTransaction& txn, RTAnyType type, const std::string& prop_name) { + switch (type.type_enum_) { + case RTAnyType::RTAnyTypeImpl::kI64Value: + return std::make_shared>(txn, + prop_name); + case RTAnyType::RTAnyTypeImpl::kI32Value: + return std::make_shared>(txn, prop_name); + case RTAnyType::RTAnyTypeImpl::kU64Value: + return std::make_shared>(txn, + prop_name); + case RTAnyType::RTAnyTypeImpl::kStringValue: + return std::make_shared>( + txn, prop_name); + case RTAnyType::RTAnyTypeImpl::kDate32: + return std::make_shared>(txn, prop_name); + default: + LOG(FATAL) << "not implemented - " << static_cast(type.type_enum_); + } + return nullptr; +} + +std::shared_ptr create_edge_property_path_accessor( + const Context& ctx, int tag, RTAnyType type) { + switch (type.type_enum_) { + case RTAnyType::RTAnyTypeImpl::kI64Value: + return std::make_shared>(ctx, tag); + case RTAnyType::RTAnyTypeImpl::kI32Value: + return std::make_shared>(ctx, tag); + case RTAnyType::RTAnyTypeImpl::kU64Value: + return std::make_shared>(ctx, tag); + case RTAnyType::RTAnyTypeImpl::kStringValue: + return std::make_shared>(ctx, + tag); + case RTAnyType::RTAnyTypeImpl::kDate32: + return std::make_shared>(ctx, tag); + case RTAnyType::RTAnyTypeImpl::kF64Value: + return std::make_shared>(ctx, tag); + default: + LOG(FATAL) << "not implemented - " << static_cast(type.type_enum_); + } + return nullptr; +} + +std::shared_ptr create_edge_label_path_accessor(const Context& ctx, + int tag) { + return std::make_shared(ctx, tag); +} + +std::shared_ptr create_edge_property_edge_accessor(RTAnyType type) { + switch (type.type_enum_) { + case RTAnyType::RTAnyTypeImpl::kI64Value: + return std::make_shared>(); + case RTAnyType::RTAnyTypeImpl::kI32Value: + return std::make_shared>(); + case RTAnyType::RTAnyTypeImpl::kU64Value: + return std::make_shared>(); + case RTAnyType::RTAnyTypeImpl::kStringValue: + return std::make_shared>(); + case RTAnyType::RTAnyTypeImpl::kDate32: + return std::make_shared>(); + case RTAnyType::RTAnyTypeImpl::kF64Value: + return std::make_shared>(); + default: + LOG(FATAL) << "not implemented - " << static_cast(type.type_enum_); + } + return nullptr; +} + +} // namespace runtime + +} // namespace gs diff --git a/flex/engines/graph_db/runtime/common/accessors.h b/flex/engines/graph_db/runtime/common/accessors.h new file mode 100644 index 000000000000..31b6faaf9f75 --- /dev/null +++ b/flex/engines/graph_db/runtime/common/accessors.h @@ -0,0 +1,605 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RUNTIME_COMMON_ACCESSORS_H_ +#define RUNTIME_COMMON_ACCESSORS_H_ + +#include "flex/engines/graph_db/database/read_transaction.h" +#include "flex/engines/graph_db/runtime/common/columns/edge_columns.h" +#include "flex/engines/graph_db/runtime/common/columns/path_columns.h" +#include "flex/engines/graph_db/runtime/common/columns/value_columns.h" +#include "flex/engines/graph_db/runtime/common/columns/vertex_columns.h" +#include "flex/engines/graph_db/runtime/common/context.h" +#include "flex/engines/graph_db/runtime/common/rt_any.h" + +namespace gs { + +namespace runtime { + +class IAccessor { + public: + virtual ~IAccessor() = default; + virtual RTAny eval_path(size_t idx) const = 0; + virtual RTAny eval_vertex(label_t label, vid_t v, size_t idx) const { + return this->eval_path(idx); + } + virtual RTAny eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const { + return this->eval_path(idx); + } + + virtual RTAny eval_path(size_t idx, int) const { + return this->eval_path(idx); + } + virtual RTAny eval_vertex(label_t label, vid_t v, size_t idx, int) const { + return this->eval_vertex(label, v, idx); + } + virtual RTAny eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx, int) const { + return this->eval_edge(label, src, dst, data, idx); + } + + virtual bool is_optional() const { return false; } + + virtual std::string name() const { return "unknown"; } + + virtual std::shared_ptr builder() const { + // LOG(FATAL) << "not implemented for " << this->name(); + return nullptr; + } +}; + +class VertexPathAccessor : public IAccessor { + public: + using elem_t = std::pair; + + VertexPathAccessor(const Context& ctx, int tag) + : vertex_col_(*std::dynamic_pointer_cast(ctx.get(tag))) {} + + bool is_optional() const override { return vertex_col_.is_optional(); } + + elem_t typed_eval_path(size_t idx) const { + return vertex_col_.get_vertex(idx); + } + + RTAny eval_path(size_t idx) const override { + return RTAny::from_vertex(typed_eval_path(idx)); + } + + RTAny eval_path(size_t idx, int) const override { + if (!vertex_col_.has_value(idx)) { + return RTAny(RTAnyType::kNull); + } + return RTAny::from_vertex(typed_eval_path(idx)); + } + + std::shared_ptr builder() const override { + return vertex_col_.builder(); + } + + private: + const IVertexColumn& vertex_col_; +}; + +class VertexIdPathAccessor : public IAccessor { + public: + using elem_t = int64_t; + VertexIdPathAccessor(const ReadTransaction& txn, const Context& ctx, int tag) + : txn_(txn), + vertex_col_(*std::dynamic_pointer_cast(ctx.get(tag))) {} + + bool is_optional() const override { return vertex_col_.is_optional(); } + + elem_t typed_eval_path(size_t idx) const { + const auto& v = vertex_col_.get_vertex(idx); + return txn_.GetVertexId(v.first, v.second).AsInt64(); + } + + RTAny eval_path(size_t idx) const override { + return RTAny::from_int64(typed_eval_path(idx)); + } + + std::shared_ptr builder() const override { + return vertex_col_.builder(); + } + + private: + const ReadTransaction& txn_; + const IVertexColumn& vertex_col_; +}; + +class VertexGIdPathAccessor : public IAccessor { + public: + using elem_t = int64_t; + VertexGIdPathAccessor(const Context& ctx, int tag) + : vertex_col_(*std::dynamic_pointer_cast(ctx.get(tag))) {} + + bool is_optional() const override { return vertex_col_.is_optional(); } + + elem_t typed_eval_path(size_t idx) const { + const auto& v = vertex_col_.get_vertex(idx); + return encode_unique_vertex_id(v.first, v.second); + } + + RTAny eval_path(size_t idx) const override { + return RTAny::from_int64(typed_eval_path(idx)); + } + + std::shared_ptr builder() const override { + return vertex_col_.builder(); + } + + private: + const IVertexColumn& vertex_col_; +}; + +template +class VertexPropertyPathAccessor : public IAccessor { + public: + using elem_t = T; + VertexPropertyPathAccessor(const ReadTransaction& txn, const Context& ctx, + int tag, const std::string& prop_name) + : vertex_col_(*std::dynamic_pointer_cast(ctx.get(tag))) { + int label_num = txn.schema().vertex_label_num(); + property_columns_.resize(label_num, nullptr); + for (int i = 0; i < label_num; ++i) { + property_columns_[i] = dynamic_cast*>( + txn.get_vertex_property_column(static_cast(i), prop_name) + .get()); + } + } + + bool is_optional() const override { + if (vertex_col_.is_optional()) { + return true; + } + auto label_set = vertex_col_.get_labels_set(); + for (auto label : label_set) { + if (property_columns_[label] == nullptr) { + return true; + } + } + return false; + } + + elem_t typed_eval_path(size_t idx) const { + const auto& v = vertex_col_.get_vertex(idx); + auto col_ptr = property_columns_[v.first]; + if (col_ptr != nullptr) { + return property_columns_[v.first]->get_view(v.second); + } else { + return elem_t(); + } + } + + RTAny eval_path(size_t idx) const override { + auto val = TypedConverter::from_typed(typed_eval_path(idx)); + return val; + } + + RTAny eval_path(size_t idx, int) const override { + if (!vertex_col_.has_value(idx)) { + return RTAny(RTAnyType::kNull); + } + const auto& v = vertex_col_.get_vertex(idx); + auto col_ptr = property_columns_[v.first]; + if (col_ptr != nullptr) { + return TypedConverter::from_typed(col_ptr->get_view(v.second)); + } else { + return RTAny(RTAnyType::kNull); + } + } + + private: + const IVertexColumn& vertex_col_; + std::vector*> property_columns_; +}; + +class VertexLabelPathAccessor : public IAccessor { + public: + using elem_t = int32_t; + VertexLabelPathAccessor(const Context& ctx, int tag) + : vertex_col_(*std::dynamic_pointer_cast(ctx.get(tag))) {} + + elem_t typed_eval_path(size_t idx) const { + return static_cast(vertex_col_.get_vertex(idx).first); + } + + RTAny eval_path(size_t idx) const override { + return RTAny(static_cast(typed_eval_path(idx))); + } + + private: + const IVertexColumn& vertex_col_; +}; + +class VertexLabelVertexAccessor : public IAccessor { + public: + using elem_t = int64_t; + VertexLabelVertexAccessor() {} + + elem_t typed_eval_vertex(label_t label, vid_t v, size_t idx) const { + return static_cast(label); + } + + RTAny eval_path(size_t idx) const override { + LOG(FATAL) << "not supposed to reach here..."; + return RTAny(); + } + + RTAny eval_vertex(label_t label, vid_t v, size_t idx) const override { + return RTAny::from_int64(label); + } +}; +template +class ContextValueAccessor : public IAccessor { + public: + using elem_t = T; + ContextValueAccessor(const Context& ctx, int tag) + : col_(*std::dynamic_pointer_cast>(ctx.get(tag))) {} + + elem_t typed_eval_path(size_t idx) const { return col_.get_value(idx); } + + RTAny eval_path(size_t idx) const override { return col_.get_elem(idx); } + + bool is_optional() const override { return col_.is_optional(); } + + RTAny eval_path(size_t idx, int) const override { + if (!col_.has_value(idx)) { + return RTAny(RTAnyType::kNull); + } + return eval_path(idx); + } + + std::shared_ptr builder() const override { + return col_.builder(); + } + + private: + const IValueColumn& col_; +}; + +class VertexIdVertexAccessor : public IAccessor { + public: + using elem_t = int64_t; + VertexIdVertexAccessor(const ReadTransaction& txn) : txn_(txn) {} + + elem_t typed_eval_vertex(label_t label, vid_t v, size_t idx) const { + return txn_.GetVertexId(label, v).AsInt64(); + } + + RTAny eval_path(size_t idx) const override { + LOG(FATAL) << "not supposed to reach here..."; + return RTAny(); + } + + RTAny eval_vertex(label_t label, vid_t v, size_t idx) const override { + return RTAny::from_int64(typed_eval_vertex(label, v, idx)); + } + + private: + const ReadTransaction& txn_; +}; + +class VertexGIdVertexAccessor : public IAccessor { + public: + using elem_t = int64_t; + VertexGIdVertexAccessor() {} + + elem_t typed_eval_vertex(label_t label, vid_t v, size_t idx) const { + return encode_unique_vertex_id(label, v); + } + + RTAny eval_path(size_t idx) const override { + LOG(FATAL) << "not supposed to reach here..."; + return RTAny(); + } + + RTAny eval_vertex(label_t label, vid_t v, size_t idx) const override { + return RTAny::from_int64(typed_eval_vertex(label, v, idx)); + } +}; + +template +class VertexPropertyVertexAccessor : public IAccessor { + public: + using elem_t = T; + VertexPropertyVertexAccessor(const ReadTransaction& txn, + const std::string& prop_name) { + int label_num = txn.schema().vertex_label_num(); + property_columns_.resize(label_num, nullptr); + for (int i = 0; i < label_num; ++i) { + property_columns_[i] = dynamic_cast*>( + txn.get_vertex_property_column(static_cast(i), prop_name) + .get()); + } + } + + elem_t typed_eval_vertex(label_t label, vid_t v, size_t idx) const { + if (property_columns_[label] == nullptr) { + return elem_t(); + } + return property_columns_[label]->get_view(v); + } + + RTAny eval_path(size_t idx) const override { + LOG(FATAL) << "not supposed to reach here..."; + return RTAny(); + } + + RTAny eval_vertex(label_t label, vid_t v, size_t idx) const override { + if (property_columns_[label] == nullptr) { + return RTAny(); + } + return TypedConverter::from_typed(property_columns_[label]->get_view(v)); + } + + RTAny eval_vertex(label_t label, vid_t v, size_t idx, int) const override { + if (property_columns_[label] == nullptr) { + return RTAny(RTAnyType::kNull); + } + return TypedConverter::from_typed(property_columns_[label]->get_view(v)); + } + + bool is_optional() const override { + for (auto col : property_columns_) { + if (col == nullptr) { + return true; + } + } + + return false; + } + + private: + std::vector*> property_columns_; +}; + +class EdgeIdPathAccessor : public IAccessor { + public: + using elem_t = std::tuple; + EdgeIdPathAccessor(const Context& ctx, int tag) + : edge_col_(*std::dynamic_pointer_cast(ctx.get(tag))) {} + + elem_t typed_eval_path(size_t idx) const { return edge_col_.get_edge(idx); } + + RTAny eval_path(size_t idx) const override { + return RTAny::from_edge(typed_eval_path(idx)); + } + + bool is_optional() const override { return edge_col_.is_optional(); } + + RTAny eval_path(size_t idx, int) const override { + if (!edge_col_.has_value(idx)) { + return RTAny(RTAnyType::kNull); + } + return RTAny::from_edge(typed_eval_path(idx)); + } + + std::shared_ptr builder() const override { + return edge_col_.builder(); + } + + private: + const IEdgeColumn& edge_col_; +}; +template +class EdgePropertyPathAccessor : public IAccessor { + public: + using elem_t = T; + EdgePropertyPathAccessor(const Context& ctx, int tag) + : col_(*std::dynamic_pointer_cast(ctx.get(tag))) {} + + RTAny eval_path(size_t idx) const override { + const auto& e = col_.get_edge(idx); + return RTAny(std::get<3>(e)); + } + + elem_t typed_eval_path(size_t idx) const { + const auto& e = col_.get_edge(idx); + elem_t ret; + ConvertAny::to(std::get<3>(e), ret); + return ret; + } + + bool is_optional() const override { return col_.is_optional(); } + + RTAny eval_path(size_t idx, int) const override { + if (!col_.has_value(idx)) { + return RTAny(RTAnyType::kNull); + } + return eval_path(idx); + } + + std::shared_ptr builder() const override { + return col_.builder(); + } + + private: + const IEdgeColumn& col_; +}; + +class EdgeLabelPathAccessor : public IAccessor { + public: + using elem_t = int32_t; + EdgeLabelPathAccessor(const Context& ctx, int tag) + : col_(*std::dynamic_pointer_cast(ctx.get(tag))) {} + + RTAny eval_path(size_t idx) const override { + const auto& e = col_.get_edge(idx); + return RTAny(static_cast(std::get<0>(e).edge_label)); + } + + elem_t typed_eval_path(size_t idx) const { + const auto& e = col_.get_edge(idx); + return static_cast(std::get<0>(e).edge_label); + } + + std::shared_ptr builder() const override { + return col_.builder(); + } + + private: + const IEdgeColumn& col_; +}; + +template +class EdgePropertyEdgeAccessor : public IAccessor { + public: + using elem_t = T; + EdgePropertyEdgeAccessor() {} + + elem_t typed_eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const { + T ret; + ConvertAny::to(data, ret); + return ret; + } + + RTAny eval_path(size_t idx) const override { + LOG(FATAL) << "not supposed to reach here..."; + return RTAny(); + } + + RTAny eval_edge(const LabelTriplet& label, vid_t src, vid_t dst, + const Any& data, size_t idx) const override { + return RTAny(data); + } +}; + +template +class ParamAccessor : public IAccessor { + public: + using elem_t = T; + ParamAccessor(const std::map& params, + const std::string& key) { + val_ = TypedConverter::typed_from_string(params.at(key)); + } + + T typed_eval_path(size_t) const { return val_; } + T typed_eval_vertex(label_t, vid_t, size_t) const { return val_; } + T typed_eval_edge(const LabelTriplet&, vid_t, vid_t, const Any&, + size_t) const { + return val_; + } + + RTAny eval_path(size_t) const override { + return TypedConverter::from_typed(val_); + } + RTAny eval_vertex(label_t, vid_t, size_t) const override { + return TypedConverter::from_typed(val_); + } + RTAny eval_edge(const LabelTriplet&, vid_t, vid_t, const Any&, + size_t) const override { + return TypedConverter::from_typed(val_); + } + + private: + T val_; +}; + +class PathIdPathAccessor : public IAccessor { + public: + using elem_t = Path; + PathIdPathAccessor(const Context& ctx, int tag) + : path_col_(*std::dynamic_pointer_cast(ctx.get(tag))) {} + + elem_t typed_eval_path(size_t idx) const { return path_col_.get_path(idx); } + + RTAny eval_path(size_t idx) const override { return path_col_.get_elem(idx); } + + std::shared_ptr builder() const override { + return path_col_.builder(); + } + + private: + const IPathColumn& path_col_; +}; + +class PathLenPathAccessor : public IAccessor { + public: + using elem_t = int32_t; + PathLenPathAccessor(const Context& ctx, int tag) + : path_col_(*std::dynamic_pointer_cast(ctx.get(tag))) {} + + elem_t typed_eval_path(size_t idx) const { + return static_cast(path_col_.get_path(idx).len()); + } + + RTAny eval_path(size_t idx) const override { + return RTAny(static_cast(typed_eval_path(idx))); + } + + private: + const IPathColumn& path_col_; +}; + +template +class ConstAccessor : public IAccessor { + public: + using elem_t = T; + ConstAccessor(const T& val) : val_(val) {} + + T typed_eval_path(size_t) const { return val_; } + T typed_eval_vertex(label_t, vid_t, size_t) const { return val_; } + T typed_eval_edge(const LabelTriplet&, vid_t, vid_t, const Any&, + size_t) const { + return val_; + } + + RTAny eval_path(size_t) const override { + return TypedConverter::from_typed(val_); + } + + RTAny eval_vertex(label_t, vid_t, size_t) const override { + return TypedConverter::from_typed(val_); + } + + RTAny eval_edge(const LabelTriplet&, vid_t, vid_t, const Any&, + size_t) const override { + return TypedConverter::from_typed(val_); + } + + private: + T val_; +}; + +std::shared_ptr create_context_value_accessor(const Context& ctx, + int tag, + RTAnyType type); + +std::shared_ptr create_vertex_property_path_accessor( + const ReadTransaction& txn, const Context& ctx, int tag, RTAnyType type, + const std::string& prop_name); + +std::shared_ptr create_vertex_property_vertex_accessor( + const ReadTransaction& txn, RTAnyType type, const std::string& prop_name); + +std::shared_ptr create_vertex_label_path_accessor(const Context& ctx, + int tag); + +std::shared_ptr create_edge_property_path_accessor( + const Context& ctx, int tag, RTAnyType type); + +std::shared_ptr create_edge_label_path_accessor(const Context& ctx, + int tag); + +std::shared_ptr create_edge_property_edge_accessor(RTAnyType type); + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_COMMON_ACCESSORS_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/columns/edge_columns.cc b/flex/engines/graph_db/runtime/common/columns/edge_columns.cc new file mode 100644 index 000000000000..f9c0a3e1d396 --- /dev/null +++ b/flex/engines/graph_db/runtime/common/columns/edge_columns.cc @@ -0,0 +1,234 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/common/columns/edge_columns.h" + +namespace gs { + +namespace runtime { + +std::shared_ptr SDSLEdgeColumn::dup() const { + SDSLEdgeColumnBuilder builder(dir_, label_, prop_type_); + builder.reserve(edges_.size()); + for (size_t i = 0; i < edges_.size(); ++i) { + auto e = get_edge(i); + builder.push_back_opt(std::get<1>(e), std::get<2>(e), std::get<3>(e)); + } + return builder.finish(); +} + +std::shared_ptr SDSLEdgeColumn::shuffle( + const std::vector& offsets) const { + SDSLEdgeColumnBuilder builder(dir_, label_, prop_type_); + size_t new_row_num = offsets.size(); + builder.reserve(new_row_num); + + if (prop_type_ == PropertyType::kEmpty) { + for (auto off : offsets) { + const auto& e = edges_[off]; + + builder.push_back_endpoints(e.first, e.second); + } + } else { + auto& ret_props = *builder.prop_col_; + ret_props.resize(new_row_num); + for (size_t idx = 0; idx < new_row_num; ++idx) { + size_t off = offsets[idx]; + const auto& e = edges_[off]; + builder.push_back_endpoints(e.first, e.second); + ret_props.set_any(idx, prop_col_->get(off)); + } + } + + return builder.finish(); +} + +std::shared_ptr SDSLEdgeColumnBuilder::finish() { + auto ret = std::make_shared(dir_, label_, prop_type_); + ret->edges_.swap(edges_); + ret->prop_col_ = prop_col_; + return ret; +} + +std::shared_ptr BDSLEdgeColumn::dup() const { + BDSLEdgeColumnBuilder builder(label_, prop_type_); + builder.reserve(size()); + for (size_t i = 0; i < size(); ++i) { + auto e = get_edge(i); + builder.push_back_opt(std::get<1>(e), std::get<2>(e), std::get<3>(e), + std::get<4>(e)); + } + return builder.finish(); +} + +std::shared_ptr BDSLEdgeColumn::shuffle( + const std::vector& offsets) const { + BDSLEdgeColumnBuilder builder(label_, prop_type_); + size_t new_row_num = offsets.size(); + builder.reserve(new_row_num); + + auto& ret_props = *builder.prop_col_; + ret_props.resize(new_row_num); + for (size_t idx = 0; idx < new_row_num; ++idx) { + size_t off = offsets[idx]; + const auto& e = edges_[off]; + builder.push_back_endpoints(std::get<0>(e), std::get<1>(e), std::get<2>(e)); + ret_props.set_any(idx, prop_col_->get(off)); + } + + return builder.finish(); +} + +std::shared_ptr BDSLEdgeColumnBuilder::finish() { + auto ret = std::make_shared(label_, prop_type_); + ret->edges_.swap(edges_); + ret->prop_col_ = prop_col_; + return ret; +} + +std::shared_ptr SDMLEdgeColumn::dup() const { + SDMLEdgeColumnBuilder builder(dir_, edge_labels_); + builder.reserve(edges_.size()); + for (auto& e : edges_) { + auto idx = std::get<0>(e); + builder.push_back_opt(idx, std::get<1>(e), std::get<2>(e), + prop_cols_[idx]->get(std::get<3>(e))); + } + return builder.finish(); +} + +std::shared_ptr SDMLEdgeColumn::shuffle( + const std::vector& offsets) const { + SDMLEdgeColumnBuilder builder(dir_, edge_labels_); + size_t new_row_num = offsets.size(); + builder.reserve(new_row_num); + for (size_t idx = 0; idx < new_row_num; ++idx) { + size_t off = offsets[idx]; + const auto& e = edges_[off]; + auto index = std::get<0>(e); + auto offset = std::get<3>(e); + builder.push_back_opt(index, std::get<1>(e), std::get<2>(e), + prop_cols_[index]->get(offset)); + } + return builder.finish(); +} + +std::shared_ptr SDMLEdgeColumnBuilder::finish() { + auto ret = std::make_shared(dir_, edge_labels_); + ret->edges_.swap(edges_); + ret->prop_cols_.swap(prop_cols_); + return ret; +} + +std::shared_ptr BDMLEdgeColumn::dup() const { + BDMLEdgeColumnBuilder builder(labels_); + builder.reserve(edges_.size()); + for (auto& e : edges_) { + auto idx = std::get<0>(e); + auto dir = std::get<4>(e) ? Direction::kOut : Direction::kIn; + builder.push_back_opt(idx, std::get<1>(e), std::get<2>(e), + prop_cols_[idx]->get(std::get<3>(e)), dir); + } + return builder.finish(); +} + +std::shared_ptr BDMLEdgeColumn::shuffle( + const std::vector& offsets) const { + BDMLEdgeColumnBuilder builder(labels_); + size_t new_row_num = offsets.size(); + builder.reserve(new_row_num); + for (size_t idx = 0; idx < new_row_num; ++idx) { + size_t off = offsets[idx]; + const auto& e = edges_[off]; + auto index = std::get<0>(e); + auto offset = std::get<3>(e); + auto dir = std::get<4>(e) ? Direction::kOut : Direction::kIn; + builder.push_back_opt(index, std::get<1>(e), std::get<2>(e), + prop_cols_[index]->get(offset), dir); + } + return builder.finish(); +} + +std::shared_ptr BDMLEdgeColumnBuilder::finish() { + auto ret = std::make_shared(labels_); + ret->edges_.swap(edges_); + ret->prop_cols_.swap(prop_cols_); + return ret; +} + +std::shared_ptr OptionalBDSLEdgeColumn::shuffle( + const std::vector& offsets) const { + OptionalBDSLEdgeColumnBuilder builder(label_, prop_type_); + size_t new_row_num = offsets.size(); + builder.reserve(new_row_num); + for (size_t idx = 0; idx < new_row_num; ++idx) { + size_t off = offsets[idx]; + const auto& e = get_edge(off); + builder.push_back_opt(std::get<1>(e), std::get<2>(e), std::get<3>(e), + std::get<4>(e)); + } + return builder.finish(); +} +std::shared_ptr OptionalBDSLEdgeColumn::dup() const { + OptionalBDSLEdgeColumnBuilder builder(label_, prop_type_); + builder.reserve(edges_.size()); + for (size_t i = 0; i < edges_.size(); ++i) { + auto e = get_edge(i); + builder.push_back_opt(std::get<1>(e), std::get<2>(e), std::get<3>(e), + std::get<4>(e)); + } + return builder.finish(); +} + +std::shared_ptr OptionalBDSLEdgeColumnBuilder::finish() { + auto ret = std::make_shared(label_, prop_type_); + ret->edges_.swap(edges_); + ret->prop_col_ = prop_col_; + return ret; +} + +std::shared_ptr OptionalSDSLEdgeColumn::shuffle( + const std::vector& offsets) const { + OptionalSDSLEdgeColumnBuilder builder(dir_, label_, prop_type_); + size_t new_row_num = offsets.size(); + builder.reserve(new_row_num); + for (size_t idx = 0; idx < new_row_num; ++idx) { + size_t off = offsets[idx]; + const auto& e = get_edge(off); + builder.push_back_opt(std::get<1>(e), std::get<2>(e), std::get<3>(e)); + } + return builder.finish(); +} + +std::shared_ptr OptionalSDSLEdgeColumn::dup() const { + OptionalSDSLEdgeColumnBuilder builder(dir_, label_, prop_type_); + builder.reserve(edges_.size()); + for (size_t i = 0; i < edges_.size(); ++i) { + auto e = get_edge(i); + builder.push_back_opt(std::get<1>(e), std::get<2>(e), std::get<3>(e)); + } + return builder.finish(); +} + +std::shared_ptr OptionalSDSLEdgeColumnBuilder::finish() { + auto ret = std::make_shared(dir_, label_, prop_type_); + ret->edges_.swap(edges_); + ret->prop_col_ = prop_col_; + return ret; +} + +} // namespace runtime + +} // namespace gs diff --git a/flex/engines/graph_db/runtime/common/columns/edge_columns.h b/flex/engines/graph_db/runtime/common/columns/edge_columns.h new file mode 100644 index 000000000000..cc58291ab636 --- /dev/null +++ b/flex/engines/graph_db/runtime/common/columns/edge_columns.h @@ -0,0 +1,878 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RUNTIME_COMMON_COLUMNS_EDGE_COLUMNS_H_ +#define RUNTIME_COMMON_COLUMNS_EDGE_COLUMNS_H_ + +#include "flex/engines/graph_db/runtime/common/columns/i_context_column.h" +#include "flex/utils/property/column.h" + +namespace gs { + +namespace runtime { + +enum class EdgeColumnType { kSDSL, kSDML, kBDSL, kBDML, kUnKnown }; +class IEdgeColumn : public IContextColumn { + public: + IEdgeColumn() = default; + virtual ~IEdgeColumn() = default; + + ContextColumnType column_type() const override { + return ContextColumnType::kEdge; + } + + virtual std::tuple get_edge( + size_t idx) const = 0; + + RTAny get_elem(size_t idx) const override { + return RTAny::from_edge(this->get_edge(idx)); + } + + RTAnyType elem_type() const override { return RTAnyType::kEdge; } + virtual std::vector get_labels() const = 0; + virtual EdgeColumnType edge_column_type() const = 0; +}; + +class SDSLEdgeColumnBuilder; +class OptionalSDSLEdgeColumnBuilder; + +class SDSLEdgeColumn : public IEdgeColumn { + public: + SDSLEdgeColumn(Direction dir, const LabelTriplet& label, + PropertyType prop_type) + : dir_(dir), + label_(label), + prop_type_(prop_type), + prop_col_(CreateColumn(prop_type, StorageStrategy::kMem)) { + prop_col_->open_in_memory(""); + } + + std::tuple get_edge( + size_t idx) const override { + return std::make_tuple(label_, edges_[idx].first, edges_[idx].second, + prop_col_->get(idx), dir_); + } + + size_t size() const override { return edges_.size(); } + + Direction dir() const { return dir_; } + + std::shared_ptr builder() const override { + return std::dynamic_pointer_cast( + std::make_shared(dir_, label_, prop_type_)); + } + + std::shared_ptr optional_builder() + const override { + return std::dynamic_pointer_cast( + std::make_shared(dir_, label_, + prop_type_)); + } + + void generate_dedup_offset(std::vector& offsets) const override { + // TODO: dedup with property value + std::vector origin_offsets(size()); + for (size_t i = 0; i < size(); ++i) { + origin_offsets[i] = i; + } + std::sort(origin_offsets.begin(), origin_offsets.end(), + [this](size_t a, size_t b) { + auto& e1 = edges_[a]; + auto& e2 = edges_[b]; + if (e1.first == e2.first) { + return e1.second < e2.second; + } + return e1.first < e2.first; + }); + + for (size_t i = 0; i < size(); ++i) { + if (i == 0 || + edges_[origin_offsets[i]] != edges_[origin_offsets[i - 1]]) { + offsets.push_back(origin_offsets[i]); + } + } + } + + ISigColumn* generate_signature() const override { + // TODO: dedup with property value + std::map, size_t> edge_map; + std::vector sigs; + for (size_t i = 0; i < size(); ++i) { + if (edge_map.find(edges_[i]) == edge_map.end()) { + edge_map[edges_[i]] = i; + } + sigs.push_back(edge_map[edges_[i]]); + } + return new SigColumn(sigs); + } + + std::string column_info() const override { + return "SDSLEdgeColumn: label = " + label_.to_string() + + ", dir = " + std::to_string((int) dir_) + + ", size = " + std::to_string(edges_.size()); + } + + std::shared_ptr shuffle( + const std::vector& offsets) const override; + + std::shared_ptr dup() const override; + + template + void foreach_edge(const FUNC_T& func) const { + if (prop_type_ == PropertyType::kEmpty) { + size_t idx = 0; + for (auto& e : edges_) { + func(idx++, label_, e.first, e.second, grape::EmptyType(), dir_); + } + } else { + size_t idx = 0; + for (auto& e : edges_) { + func(idx, label_, e.first, e.second, prop_col_->get(idx), dir_); + ++idx; + } + } + } + + std::vector get_labels() const override { return {label_}; } + + EdgeColumnType edge_column_type() const override { + return EdgeColumnType::kSDSL; + } + + private: + friend class SDSLEdgeColumnBuilder; + Direction dir_; + LabelTriplet label_; + std::vector> edges_; + PropertyType prop_type_; + std::shared_ptr prop_col_; +}; + +class OptionalSDSLEdgeColumn : public IEdgeColumn { + public: + OptionalSDSLEdgeColumn(Direction dir, const LabelTriplet& label, + PropertyType prop_type) + : dir_(dir), + label_(label), + prop_type_(prop_type), + prop_col_(CreateColumn(prop_type, StorageStrategy::kMem)) { + prop_col_->open_in_memory(""); + } + + std::tuple get_edge( + size_t idx) const override { + return std::make_tuple(label_, edges_[idx].first, edges_[idx].second, + prop_col_->get(idx), dir_); + } + + size_t size() const override { return edges_.size(); } + + void generate_dedup_offset(std::vector& offsets) const override { + std::vector origin_offsets(size()); + for (size_t i = 0; i < size(); ++i) { + origin_offsets[i] = i; + } + std::sort(origin_offsets.begin(), origin_offsets.end(), + [this](size_t a, size_t b) { + auto& e1 = edges_[a]; + auto& e2 = edges_[b]; + if (e1.first == e2.first) { + return e1.second < e2.second; + } + return e1.first < e2.first; + }); + + for (size_t i = 0; i < size(); ++i) { + if (i == 0 || + edges_[origin_offsets[i]] != edges_[origin_offsets[i - 1]]) { + offsets.push_back(origin_offsets[i]); + } + } + } + + std::shared_ptr builder() const override { + return std::dynamic_pointer_cast( + std::make_shared(dir_, label_, + prop_type_)); + } + + ISigColumn* generate_signature() const override { + std::map, size_t> edge_map; + std::vector sigs; + for (size_t i = 0; i < size(); ++i) { + if (edge_map.find(edges_[i]) == edge_map.end()) { + edge_map[edges_[i]] = i; + } + sigs.push_back(edge_map[edges_[i]]); + } + return new SigColumn(sigs); + } + + std::string column_info() const override { + return "OptionalSDSLEdgeColumn: label = " + label_.to_string() + + ", dir = " + std::to_string((int) dir_) + + ", size = " + std::to_string(edges_.size()); + } + + std::shared_ptr shuffle( + const std::vector& offsets) const override; + + std::shared_ptr dup() const override; + + template + void foreach_edge(const FUNC_T& func) const { + if (prop_type_ == PropertyType::kEmpty) { + size_t idx = 0; + for (auto& e : edges_) { + func(idx++, label_, e.first, e.second, grape::EmptyType(), dir_, 0); + } + } else { + size_t idx = 0; + for (auto& e : edges_) { + func(idx, label_, e.first, e.second, prop_col_->get(idx), dir_, 0); + ++idx; + } + } + } + + bool is_optional() const override { return true; } + + bool has_value(size_t idx) const override { + return edges_[idx].first != std::numeric_limits::max() && + edges_[idx].second != std::numeric_limits::max(); + } + + std::vector get_labels() const override { + LOG(INFO) << "get_labels: " << label_.to_string() << std::endl; + return {label_}; + } + + EdgeColumnType edge_column_type() const override { + return EdgeColumnType::kSDSL; + } + + private: + friend class OptionalSDSLEdgeColumnBuilder; + Direction dir_; + LabelTriplet label_; + std::vector> edges_; + PropertyType prop_type_; + std::shared_ptr prop_col_; +}; + +class OptionalSDSLEdgeColumnBuilder : public IOptionalContextColumnBuilder { + public: + OptionalSDSLEdgeColumnBuilder(Direction dir, const LabelTriplet& label, + PropertyType prop_type) + : dir_(dir), + label_(label), + prop_type_(prop_type), + prop_col_(CreateColumn(prop_type, StorageStrategy::kMem)) { + prop_col_->open_in_memory(""); + } + ~OptionalSDSLEdgeColumnBuilder() = default; + + void reserve(size_t size) override { edges_.reserve(size); } + void push_back_elem(const RTAny& val) override { + const auto& e = val.as_edge(); + push_back_opt(std::get<1>(e), std::get<2>(e), std::get<3>(e)); + } + void push_back_opt(vid_t src, vid_t dst, const Any& data) { + edges_.emplace_back(src, dst); + size_t len = edges_.size(); + prop_col_->resize(len); + prop_col_->set_any(len - 1, data); + } + + void push_back_null() override { + edges_.emplace_back(std::numeric_limits::max(), + std::numeric_limits::max()); + } + void push_back_endpoints(vid_t src, vid_t dst) { + edges_.emplace_back(src, dst); + } + + std::shared_ptr finish() override; + + private: + friend class OptionalSDSLEdgeColumn; + Direction dir_; + LabelTriplet label_; + std::vector> edges_; + PropertyType prop_type_; + std::shared_ptr prop_col_; +}; + +class BDSLEdgeColumnBuilder; + +class OptionalBDSLEdgeColumnBuilder; +class BDSLEdgeColumn : public IEdgeColumn { + public: + BDSLEdgeColumn(const LabelTriplet& label, PropertyType prop_type) + : label_(label), + prop_type_(prop_type), + prop_col_(CreateColumn(prop_type, StorageStrategy::kMem)) { + prop_col_->open_in_memory(""); + } + + std::tuple get_edge( + size_t idx) const override { + auto src = std::get<0>(edges_[idx]); + auto dst = std::get<1>(edges_[idx]); + auto dir = std::get<2>(edges_[idx]); + return std::make_tuple(label_, src, dst, prop_col_->get(idx), + (dir ? Direction::kOut : Direction::kIn)); + } + + size_t size() const override { return edges_.size(); } + + std::string column_info() const override { + return "BDSLEdgeColumn: label = " + label_.to_string() + + ", size = " + std::to_string(edges_.size()); + } + + std::shared_ptr builder() const override { + return std::dynamic_pointer_cast( + std::make_shared(label_, prop_type_)); + } + + std::shared_ptr optional_builder() + const override { + return std::dynamic_pointer_cast( + std::make_shared(label_, prop_type_)); + } + std::shared_ptr shuffle( + const std::vector& offsets) const override; + + std::shared_ptr dup() const override; + + template + void foreach_edge(const FUNC_T& func) const { + size_t idx = 0; + for (auto& e : edges_) { + func(idx, label_, std::get<0>(e), std::get<1>(e), prop_col_->get(idx), + (std::get<2>(e) ? Direction::kOut : Direction::kIn)); + ++idx; + } + } + + std::vector get_labels() const override { return {label_}; } + + EdgeColumnType edge_column_type() const override { + return EdgeColumnType::kBDSL; + } + + private: + friend class BDSLEdgeColumnBuilder; + LabelTriplet label_; + std::vector> edges_; + PropertyType prop_type_; + std::shared_ptr prop_col_; +}; + +class OptionalBDSLEdgeColumn : public IEdgeColumn { + public: + OptionalBDSLEdgeColumn(const LabelTriplet& label, PropertyType prop_type) + : label_(label), + prop_type_(prop_type), + prop_col_(CreateColumn(prop_type, StorageStrategy::kMem)) { + prop_col_->open_in_memory(""); + } + + std::tuple get_edge( + size_t idx) const override { + auto src = std::get<0>(edges_[idx]); + auto dst = std::get<1>(edges_[idx]); + auto dir = std::get<2>(edges_[idx]); + return std::make_tuple(label_, src, dst, prop_col_->get(idx), + (dir ? Direction::kOut : Direction::kIn)); + } + + size_t size() const override { return edges_.size(); } + + std::shared_ptr builder() const override { + return std::dynamic_pointer_cast( + std::make_shared(label_, prop_type_)); + } + + std::string column_info() const override { + return "OptionalBDSLEdgeColumn: label = " + label_.to_string() + + ", size = " + std::to_string(edges_.size()); + } + + std::shared_ptr shuffle( + const std::vector& offsets) const override; + + std::shared_ptr dup() const override; + + template + void foreach_edge(const FUNC_T& func) const { + size_t idx = 0; + for (auto& e : edges_) { + func(idx, label_, std::get<0>(e), std::get<1>(e), prop_col_->get(idx), + (std::get<2>(e) ? Direction::kOut : Direction::kIn)); + ++idx; + } + } + + bool is_optional() const override { return true; } + + bool has_value(size_t idx) const override { + return std::get<0>(edges_[idx]) != std::numeric_limits::max() && + std::get<1>(edges_[idx]) != std::numeric_limits::max(); + } + + std::vector get_labels() const override { + LOG(INFO) << "get_labels: " << label_.to_string() << std::endl; + return {label_}; + } + + EdgeColumnType edge_column_type() const override { + return EdgeColumnType::kBDSL; + } + + private: + friend class OptionalBDSLEdgeColumnBuilder; + LabelTriplet label_; + std::vector> edges_; + PropertyType prop_type_; + std::shared_ptr prop_col_; +}; + +class SDMLEdgeColumnBuilder; + +class SDMLEdgeColumn : public IEdgeColumn { + public: + SDMLEdgeColumn( + Direction dir, + const std::vector>& labels) + : dir_(dir) { + size_t idx = 0; + prop_cols_.resize(labels.size()); + for (const auto& label : labels) { + edge_labels_.emplace_back(label); + index_[label.first] = idx++; + prop_cols_[index_[label.first]] = + CreateColumn(label.second, StorageStrategy::kMem); + prop_cols_[index_[label.first]]->open_in_memory(""); + } + } + + std::tuple get_edge( + size_t idx) const override { + auto& e = edges_[idx]; + auto index = std::get<0>(e); + auto label = edge_labels_[index].first; + auto offset = std::get<3>(e); + return std::tuple( + label, std::get<1>(e), std::get<2>(e), prop_cols_[index]->get(offset), + dir_); + } + + size_t size() const override { return edges_.size(); } + + std::shared_ptr builder() const override { + return std::dynamic_pointer_cast( + std::make_shared(dir_, edge_labels_)); + } + + std::string column_info() const override { + std::stringstream ss{}; + + for (size_t idx = 0; idx < edge_labels_.size(); ++idx) { + auto label = edge_labels_[idx]; + if (idx != 0) { + ss << ", "; + } + ss << label.first.to_string(); + } + return "SDMLEdgeColumn: label = {" + ss.str() + + "}, dir = " + std::to_string((int) dir_) + + ", size = " + std::to_string(edges_.size()); + } + + std::shared_ptr shuffle( + const std::vector& offsets) const override; + + std::shared_ptr dup() const override; + + template + void foreach_edge(const FUNC_T& func) const { + size_t idx = 0; + for (auto& e : edges_) { + auto index = std::get<0>(e); + auto label = edge_labels_[index].first; + auto offset = std::get<3>(e); + func(idx, label, std::get<1>(e), std::get<2>(e), + prop_cols_[index]->get(offset), dir_); + ++idx; + } + } + + std::vector get_labels() const override { + std::vector labels; + for (auto& label : edge_labels_) { + labels.push_back(label.first); + } + return labels; + } + + Direction dir() const { return dir_; } + EdgeColumnType edge_column_type() const override { + return EdgeColumnType::kSDML; + } + + private: + friend class SDMLEdgeColumnBuilder; + Direction dir_; + std::map index_; + std::vector> edge_labels_; + std::vector> edges_; + std::vector> prop_cols_; +}; + +class BDMLEdgeColumnBuilder; + +class BDMLEdgeColumn : public IEdgeColumn { + public: + BDMLEdgeColumn( + const std::vector>& labels) + : labels_(labels) { + size_t idx = 0; + prop_cols_.resize(labels.size()); + for (const auto& label : labels) { + index_[label.first] = idx++; + prop_cols_[index_[label.first]] = + CreateColumn(label.second, StorageStrategy::kMem); + prop_cols_[index_[label.first]]->open_in_memory(""); + } + } + + std::tuple get_edge( + size_t idx) const override { + auto& e = edges_[idx]; + auto index = std::get<0>(e); + auto label = labels_[index].first; + auto offset = std::get<3>(e); + return std::tuple( + label, std::get<1>(e), std::get<2>(e), prop_cols_[index]->get(offset), + (std::get<4>(e) ? Direction::kOut : Direction::kIn)); + } + + size_t size() const override { return edges_.size(); } + + std::shared_ptr builder() const override { + return std::dynamic_pointer_cast( + std::make_shared(labels_)); + } + + std::string column_info() const override { + std::stringstream ss{}; + + for (size_t idx = 0; idx < labels_.size(); ++idx) { + auto label = labels_[idx]; + if (idx != 0) { + ss << ", "; + } + ss << label.first.to_string(); + } + return "BDMLEdgeColumn: label = {" + ss.str() + + "}, size = " + std::to_string(edges_.size()); + } + + std::shared_ptr shuffle( + const std::vector& offsets) const override; + + std::shared_ptr dup() const override; + + template + void foreach_edge(const FUNC_T& func) const { + size_t idx = 0; + for (auto& e : edges_) { + auto index = std::get<0>(e); + auto label = labels_[index].first; + auto offset = std::get<3>(e); + func(idx, label, std::get<1>(e), std::get<2>(e), + prop_cols_[index]->get(offset), + (std::get<4>(e) ? Direction::kOut : Direction::kIn)); + ++idx; + } + } + + std::vector get_labels() const override { + std::vector labels; + for (auto& label : labels_) { + labels.push_back(label.first); + } + return labels; + } + + EdgeColumnType edge_column_type() const override { + return EdgeColumnType::kBDML; + } + + private: + friend class BDMLEdgeColumnBuilder; + std::map index_; + std::vector> labels_; + std::vector> edges_; + std::vector> prop_cols_; +}; + +class SDSLEdgeColumnBuilder : public IContextColumnBuilder { + public: + SDSLEdgeColumnBuilder(Direction dir, const LabelTriplet& label, + PropertyType prop_type) + : dir_(dir), + label_(label), + prop_type_(prop_type), + prop_col_(CreateColumn(prop_type, StorageStrategy::kMem)) { + prop_col_->open_in_memory(""); + } + ~SDSLEdgeColumnBuilder() = default; + + void reserve(size_t size) override { edges_.reserve(size); } + void push_back_elem(const RTAny& val) override { + const auto& e = val.as_edge(); + push_back_opt(std::get<1>(e), std::get<2>(e), std::get<3>(e)); + } + void push_back_opt(vid_t src, vid_t dst, const Any& data) { + edges_.emplace_back(src, dst); + size_t len = edges_.size(); + prop_col_->resize(len); + prop_col_->set_any(len - 1, data); + } + void push_back_endpoints(vid_t src, vid_t dst) { + edges_.emplace_back(src, dst); + } + + std::shared_ptr finish() override; + + private: + friend class SDSLEdgeColumn; + Direction dir_; + LabelTriplet label_; + std::vector> edges_; + PropertyType prop_type_; + std::shared_ptr prop_col_; +}; + +class BDSLEdgeColumnBuilder : public IContextColumnBuilder { + public: + BDSLEdgeColumnBuilder(const LabelTriplet& label, PropertyType prop_type) + : label_(label), + prop_type_(prop_type), + prop_col_(CreateColumn(prop_type, StorageStrategy::kMem)) { + prop_col_->open_in_memory(""); + } + ~BDSLEdgeColumnBuilder() = default; + + void reserve(size_t size) override { edges_.reserve(size); } + void push_back_elem(const RTAny& val) override { + const auto& e = val.as_edge(); + push_back_opt(std::get<1>(e), std::get<2>(e), std::get<3>(e), + std::get<4>(e)); + } + void push_back_opt(vid_t src, vid_t dst, const Any& data, Direction dir) { + edges_.emplace_back(src, dst, dir == Direction::kOut); + size_t len = edges_.size(); + prop_col_->resize(len); + prop_col_->set_any(len - 1, data); + } + void push_back_endpoints(vid_t src, vid_t dst, Direction dir) { + edges_.emplace_back(src, dst, dir == Direction::kOut); + } + + void push_back_endpoints(vid_t src, vid_t dst, bool dir) { + edges_.emplace_back(src, dst, dir); + } + + std::shared_ptr finish() override; + + private: + friend class BDSLEdgeColumn; + LabelTriplet label_; + std::vector> edges_; + PropertyType prop_type_; + std::shared_ptr prop_col_; +}; +class SDMLEdgeColumnBuilder : public IContextColumnBuilder { + public: + SDMLEdgeColumnBuilder( + Direction dir, + const std::vector>& labels) + : dir_(dir) { + size_t idx = 0; + prop_cols_.resize(labels.size()); + for (const auto& label : labels) { + edge_labels_.emplace_back(label); + index_[label.first] = idx++; + prop_cols_[index_[label.first]] = + CreateColumn(label.second, StorageStrategy::kMem); + prop_cols_[index_[label.first]]->open_in_memory(""); + } + } + ~SDMLEdgeColumnBuilder() = default; + + void reserve(size_t size) override { edges_.reserve(size); } + void push_back_elem(const RTAny& val) override { + const auto& e = val.as_edge(); + auto label = std::get<0>(e); + auto index = index_[label]; + push_back_opt(index, std::get<1>(e), std::get<2>(e), std::get<3>(e)); + } + void push_back_opt(int8_t index, vid_t src, vid_t dst, const Any& data) { + edges_.emplace_back(index, src, dst, prop_cols_[index]->size()); + prop_cols_[index]->resize(prop_cols_[index]->size() + 1); + prop_cols_[index]->set_any(prop_cols_[index]->size() - 1, data); + } + + void push_back_opt(LabelTriplet label, vid_t src, vid_t dst, + const Any& data) { + auto index = index_[label]; + push_back_opt(index, src, dst, data); + } + + void push_back_endpoints(int8_t index, vid_t src, vid_t dst) { + edges_.emplace_back(index, src, dst, prop_cols_[index]->size()); + } + + std::shared_ptr finish() override; + + private: + friend class SDMLEdgeColumn; + Direction dir_; + std::map index_; + std::vector> edge_labels_; + std::vector> edges_; + std::vector> prop_cols_; +}; + +class BDMLEdgeColumnBuilder : public IContextColumnBuilder { + public: + BDMLEdgeColumnBuilder() : index_(), labels_() {} + BDMLEdgeColumnBuilder( + const std::vector>& labels) + : labels_(labels) { + size_t idx = 0; + prop_cols_.resize(labels.size()); + for (const auto& label : labels) { + index_[label.first] = idx++; + prop_cols_[index_[label.first]] = + CreateColumn(label.second, StorageStrategy::kMem); + prop_cols_[index_[label.first]]->open_in_memory(""); + } + } + ~BDMLEdgeColumnBuilder() = default; + + void reserve(size_t size) override { edges_.reserve(size); } + void push_back_elem(const RTAny& val) override { + const auto& e = val.as_edge(); + auto label = std::get<0>(e); + if (index_.find(label) == index_.end()) { + index_[label] = labels_.size(); + auto data = std::get<3>(e); + labels_.emplace_back(label, data.type); + prop_cols_.emplace_back(CreateColumn(data.type, StorageStrategy::kMem)); + prop_cols_.back()->open_in_memory(""); + } + auto index = index_[label]; + push_back_opt(index, std::get<1>(e), std::get<2>(e), std::get<3>(e), + std::get<4>(e)); + } + void push_back_opt(int8_t index, vid_t src, vid_t dst, const Any& data, + Direction dir) { + edges_.emplace_back(index, src, dst, prop_cols_[index]->size(), + dir == Direction::kOut); + prop_cols_[index]->resize(prop_cols_[index]->size() + 1); + prop_cols_[index]->set_any(prop_cols_[index]->size() - 1, data); + } + + void push_back_opt(LabelTriplet label, vid_t src, vid_t dst, const Any& data, + Direction dir) { + auto index = index_[label]; + push_back_opt(index, src, dst, data, dir); + } + + void push_back_endpoints(int8_t index, vid_t src, vid_t dst, Direction dir) { + edges_.emplace_back(index, src, dst, prop_cols_[index]->size(), + dir == Direction::kOut); + } + + void push_back_endpoints(int8_t index, vid_t src, vid_t dst, bool dir) { + edges_.emplace_back(index, src, dst, prop_cols_[index]->size(), dir); + } + + std::shared_ptr finish() override; + + private: + friend class BDMLEdgeColumn; + + std::map index_; + std::vector> labels_; + std::vector> edges_; + std::vector> prop_cols_; +}; + +class OptionalBDSLEdgeColumnBuilder : public IOptionalContextColumnBuilder { + public: + OptionalBDSLEdgeColumnBuilder(const LabelTriplet& label, + PropertyType prop_type) + : label_(label), + prop_type_(prop_type), + prop_col_(CreateColumn(prop_type, StorageStrategy::kMem)) { + prop_col_->open_in_memory(""); + } + ~OptionalBDSLEdgeColumnBuilder() = default; + + void reserve(size_t size) override { edges_.reserve(size); } + void push_back_elem(const RTAny& val) override { + const auto& e = val.as_edge(); + push_back_opt(std::get<1>(e), std::get<2>(e), std::get<3>(e), + std::get<4>(e)); + } + void push_back_opt(vid_t src, vid_t dst, const Any& data, Direction dir) { + edges_.emplace_back(src, dst, dir == Direction::kOut); + size_t len = edges_.size(); + prop_col_->resize(len); + prop_col_->set_any(len - 1, data); + } + void push_back_endpoints(vid_t src, vid_t dst, Direction dir) { + edges_.emplace_back(src, dst, dir == Direction::kOut); + } + + void push_back_endpoints(vid_t src, vid_t dst, bool dir) { + edges_.emplace_back(src, dst, dir); + } + + void push_back_null() override { + edges_.emplace_back(std::numeric_limits::max(), + std::numeric_limits::max(), false); + } + + std::shared_ptr finish() override; + + private: + friend class OptionalBDSLEdgeColumn; + LabelTriplet label_; + std::vector> edges_; + PropertyType prop_type_; + std::shared_ptr prop_col_; +}; + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_COMMON_COLUMNS_EDGE_COLUMNS_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/columns/i_context_column.h b/flex/engines/graph_db/runtime/common/columns/i_context_column.h new file mode 100644 index 000000000000..c35f97f19378 --- /dev/null +++ b/flex/engines/graph_db/runtime/common/columns/i_context_column.h @@ -0,0 +1,243 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RUNTIME_COMMON_COLUMNS_I_CONTEXT_COLUMNS_H_ +#define RUNTIME_COMMON_COLUMNS_I_CONTEXT_COLUMNS_H_ + +#include +#include + +#include "flex/engines/graph_db/runtime/common/rt_any.h" + +#include "glog/logging.h" + +namespace gs { + +namespace runtime { + +enum class ContextColumnType { + kVertex, + kEdge, + kValue, + kPath, + kOptionalValue, +}; + +class ISigColumn { + public: + ISigColumn() = default; + virtual ~ISigColumn() = default; + virtual size_t get_sig(size_t idx) const = 0; +}; + +template +class SigColumn : public ISigColumn { + public: + SigColumn(const std::vector& data) : data_(data.data()) {} + ~SigColumn() = default; + size_t get_sig(size_t idx) const override { + return static_cast(data_[idx]); + } + + private: + const T* data_; +}; + +template <> +class SigColumn : public ISigColumn { + public: + SigColumn(const std::vector& data) : data_(data.data()) {} + ~SigColumn() = default; + size_t get_sig(size_t idx) const override { + return static_cast(data_[idx].milli_second); + } + + private: + const Date* data_; +}; + +template <> +class SigColumn> : public ISigColumn { + public: + SigColumn(const std::vector>& data) + : data_(data.data()) {} + ~SigColumn() = default; + size_t get_sig(size_t idx) const override { + const auto& v = data_[idx]; + size_t ret = v.first; + ret <<= 32; + ret += v.second; + return ret; + } + + private: + const std::pair* data_; +}; + +template <> +class SigColumn : public ISigColumn { + public: + SigColumn(const std::vector& data) { + std::unordered_map table; + sig_list_.reserve(data.size()); + for (auto& str : data) { + auto iter = table.find(str); + if (iter == table.end()) { + size_t idx = table.size(); + table.emplace(str, idx); + sig_list_.push_back(idx); + } else { + sig_list_.push_back(iter->second); + } + } + } + ~SigColumn() = default; + size_t get_sig(size_t idx) const override { return sig_list_[idx]; } + + private: + std::vector sig_list_; +}; + +template <> +class SigColumn> : public ISigColumn { + public: + SigColumn(const std::vector>& data) { + std::map, size_t> table; + sig_list_.reserve(data.size()); + for (auto& str : data) { + auto iter = table.find(str); + if (iter == table.end()) { + size_t idx = table.size(); + table.emplace(str, idx); + sig_list_.push_back(idx); + } else { + sig_list_.push_back(iter->second); + } + } + } + ~SigColumn() = default; + size_t get_sig(size_t idx) const override { return sig_list_[idx]; } + + private: + std::vector sig_list_; +}; + +template <> +class SigColumn> : public ISigColumn { + public: + SigColumn(const std::vector>& data) { + std::map, size_t> table; + sig_list_.reserve(data.size()); + for (auto& str : data) { + auto iter = table.find(str); + if (iter == table.end()) { + size_t idx = table.size(); + table.emplace(str, idx); + sig_list_.push_back(idx); + } else { + sig_list_.push_back(iter->second); + } + } + } + ~SigColumn() = default; + size_t get_sig(size_t idx) const override { return sig_list_[idx]; } + + private: + std::vector sig_list_; +}; + +class IContextColumnBuilder; +class IOptionalContextColumnBuilder; + +class IContextColumn { + public: + IContextColumn() = default; + virtual ~IContextColumn() = default; + + virtual size_t size() const { + LOG(FATAL) << "not implemented for " << this->column_info(); + return 0; + } + + virtual std::shared_ptr dup() const = 0; + + virtual std::string column_info() const = 0; + virtual ContextColumnType column_type() const = 0; + + virtual RTAnyType elem_type() const = 0; + + virtual std::shared_ptr builder() const = 0; + + virtual std::shared_ptr optional_builder() + const { + LOG(FATAL) << "not implemented for " << this->column_info(); + return nullptr; + } + + virtual std::shared_ptr shuffle( + const std::vector& offsets) const { + LOG(FATAL) << "not implemented for " << this->column_info(); + return nullptr; + } + + virtual std::shared_ptr union_col( + std::shared_ptr other) const { + LOG(FATAL) << "not implemented for " << this->column_info(); + return nullptr; + } + + virtual RTAny get_elem(size_t idx) const { + LOG(FATAL) << "not implemented for " << this->column_info(); + return RTAny(); + } + + virtual bool has_value(size_t idx) const { return true; } + + virtual bool is_optional() const { return false; } + + virtual ISigColumn* generate_signature() const { + LOG(FATAL) << "not implemented for " << this->column_info(); + return nullptr; + } + + virtual void generate_dedup_offset(std::vector& offsets) const { + LOG(FATAL) << "not implemented for " << this->column_info(); + } +}; + +class IContextColumnBuilder { + public: + IContextColumnBuilder() = default; + virtual ~IContextColumnBuilder() = default; + + virtual void reserve(size_t size) = 0; + virtual void push_back_elem(const RTAny& val) = 0; + + virtual std::shared_ptr finish() = 0; +}; + +class IOptionalContextColumnBuilder : public IContextColumnBuilder { + public: + IOptionalContextColumnBuilder() = default; + virtual ~IOptionalContextColumnBuilder() = default; + + virtual void push_back_null() = 0; +}; + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_COMMON_COLUMNS_I_CONTEXT_COLUMNS_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/columns/path_columns.cc b/flex/engines/graph_db/runtime/common/columns/path_columns.cc new file mode 100644 index 000000000000..51618cd19e37 --- /dev/null +++ b/flex/engines/graph_db/runtime/common/columns/path_columns.cc @@ -0,0 +1,46 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/common/columns/path_columns.h" + +namespace gs { +namespace runtime { +std::shared_ptr GeneralPathColumn::dup() const { + GeneralPathColumnBuilder builder; + for (const auto& path : data_) { + builder.push_back_opt(path); + } + builder.set_path_impls(path_impls_); + return builder.finish(); +} + +std::shared_ptr GeneralPathColumn::shuffle( + const std::vector& offsets) const { + GeneralPathColumnBuilder builder; + builder.reserve(offsets.size()); + for (auto& offset : offsets) { + builder.push_back_opt(data_[offset]); + } + builder.set_path_impls(path_impls_); + return builder.finish(); +} +std::shared_ptr GeneralPathColumn::builder() const { + auto builder = std::make_shared(); + builder->set_path_impls(path_impls_); + return std::dynamic_pointer_cast(builder); +} + +} // namespace runtime +} // namespace gs \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/columns/path_columns.h b/flex/engines/graph_db/runtime/common/columns/path_columns.h new file mode 100644 index 000000000000..136febb6d152 --- /dev/null +++ b/flex/engines/graph_db/runtime/common/columns/path_columns.h @@ -0,0 +1,134 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RUNTIME_COMMON_COLUMNS_PATH_COLUMNS_H_ +#define RUNTIME_COMMON_COLUMNS_PATH_COLUMNS_H_ +#include "flex/engines/graph_db/runtime/common/columns/i_context_column.h" + +namespace gs { +namespace runtime { + +class IPathColumn : public IContextColumn { + public: + IPathColumn() = default; + virtual ~IPathColumn() = default; + virtual size_t size() const = 0; + virtual std::string column_info() const = 0; + virtual ContextColumnType column_type() const = 0; + virtual std::shared_ptr dup() const = 0; + virtual std::shared_ptr shuffle( + const std::vector& offsets) const = 0; + virtual RTAnyType elem_type() const = 0; + virtual RTAny get_elem(size_t idx) const = 0; + virtual const Path& get_path(size_t idx) const = 0; + virtual ISigColumn* generate_signature() const = 0; + virtual void generate_dedup_offset(std::vector& offsets) const = 0; +}; + +class GeneralPathColumnBuilder; + +class GeneralPathColumn : public IPathColumn { + public: + GeneralPathColumn() = default; + ~GeneralPathColumn() {} + size_t size() const override { return data_.size(); } + std::string column_info() const override { + return "GeneralPathColumn[" + std::to_string(size()) + "]"; + } + ContextColumnType column_type() const override { + return ContextColumnType::kPath; + } + std::shared_ptr dup() const override; + std::shared_ptr shuffle( + const std::vector& offsets) const override; + RTAnyType elem_type() const override { return RTAnyType::kPath; } + RTAny get_elem(size_t idx) const override { return RTAny(data_[idx]); } + const Path& get_path(size_t idx) const override { return data_[idx]; } + ISigColumn* generate_signature() const override { + LOG(FATAL) << "not implemented for " << this->column_info(); + return nullptr; + } + + std::shared_ptr builder() const override; + + void generate_dedup_offset(std::vector& offsets) const override { + std::vector origin_offsets(data_.size()); + for (size_t i = 0; i < data_.size(); ++i) { + origin_offsets[i] = i; + } + std::sort(origin_offsets.begin(), origin_offsets.end(), + [this](size_t a, size_t b) { + // data_[a] == data_[b] + if (!(data_[a] < data_[b]) && !(data_[b] < data_[a])) { + return a < b; + } + return data_[a] < data_[b]; + }); + offsets.clear(); + offsets.push_back(origin_offsets[0]); + for (size_t i = 1; i < origin_offsets.size(); ++i) { + if (!(data_[origin_offsets[i]] == data_[origin_offsets[i - 1]])) { + offsets.push_back(origin_offsets[i]); + } + } + } + + template + void foreach_path(FUNC func) const { + for (size_t i = 0; i < data_.size(); ++i) { + const auto& path = data_[i]; + func(i, path); + } + } + + void set_path_impls(std::vector>&& path_impls) { + path_impls_.swap(path_impls); + } + + private: + friend class GeneralPathColumnBuilder; + std::vector data_; + std::vector> path_impls_; +}; + +class GeneralPathColumnBuilder : public IContextColumnBuilder { + public: + GeneralPathColumnBuilder() = default; + ~GeneralPathColumnBuilder() = default; + void push_back_opt(const Path& p) { data_.push_back(p); } + void push_back_elem(const RTAny& val) override { + data_.push_back(val.as_path()); + } + void reserve(size_t size) override { data_.reserve(size); } + void set_path_impls( + const std::vector>& path_impls) { + path_impls_ = path_impls; + } + std::shared_ptr finish() override { + auto col = std::make_shared(); + col->data_.swap(data_); + col->set_path_impls(std::move(path_impls_)); + path_impls_.clear(); + return col; + } + + private: + std::vector data_; + std::vector> path_impls_; +}; + +} // namespace runtime +} // namespace gs +#endif // RUNTIME_COMMON_COLUMNS_PATH_COLUMNS_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/columns/value_columns.cc b/flex/engines/graph_db/runtime/common/columns/value_columns.cc new file mode 100644 index 000000000000..8a1c23ac8dfc --- /dev/null +++ b/flex/engines/graph_db/runtime/common/columns/value_columns.cc @@ -0,0 +1,90 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/common/columns/value_columns.h" + +namespace gs { + +namespace runtime { + +std::shared_ptr ValueColumn::shuffle( + const std::vector& offsets) const { + ValueColumnBuilder builder; + builder.reserve(offsets.size()); + for (auto offset : offsets) { + builder.push_back_opt(data_[offset]); + } + return builder.finish(); +} + +std::shared_ptr ValueColumn::dup() const { + ValueColumnBuilder builder; + for (auto v : data_) { + builder.push_back_opt(v); + } + return builder.finish(); +} + +std::shared_ptr OptionalValueColumn::shuffle( + const std::vector& offsets) const { + OptionalValueColumnBuilder builder; + for (size_t i : offsets) { + builder.push_back_opt(data_[i], valid_[i]); + } + return builder.finish(); +} + +std::shared_ptr OptionalValueColumn::dup() + const { + OptionalValueColumnBuilder builder; + for (size_t i = 0; i < data_.size(); ++i) { + builder.push_back_opt(data_[i], valid_[i]); + } + return builder.finish(); +} + +std::shared_ptr MapValueColumn::dup() const { + MapValueColumnBuilder builder; + builder.set_keys(keys_); + for (const auto& values : values_) { + builder.push_back_opt(values); + } + return builder.finish(); +} + +std::shared_ptr MapValueColumn::shuffle( + const std::vector& offsets) const { + MapValueColumnBuilder builder; + builder.reserve(offsets.size()); + builder.set_keys(keys_); + for (auto offset : offsets) { + builder.push_back_opt(values_[offset]); + } + return builder.finish(); +} + +std::shared_ptr MapValueColumn::builder() const { + auto builder = std::make_shared(); + builder->set_keys(keys_); + return builder; +} + +template class ValueColumn; +template class ValueColumn>; +template class ValueColumn>; + +} // namespace runtime + +} // namespace gs diff --git a/flex/engines/graph_db/runtime/common/columns/value_columns.h b/flex/engines/graph_db/runtime/common/columns/value_columns.h new file mode 100644 index 000000000000..bc0372bb2d49 --- /dev/null +++ b/flex/engines/graph_db/runtime/common/columns/value_columns.h @@ -0,0 +1,731 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RUNTIME_COMMON_COLUMNS_VALUE_COLUMNS_H_ +#define RUNTIME_COMMON_COLUMNS_VALUE_COLUMNS_H_ + +#include "flex/engines/graph_db/runtime/common/columns/i_context_column.h" +#include "flex/engines/graph_db/runtime/common/rt_any.h" + +#include + +namespace gs { + +namespace runtime { + +template +class ValueColumnBuilder; + +template +class OptionalValueColumnBuilder; + +template +class IValueColumn : public IContextColumn { + public: + IValueColumn() = default; + virtual ~IValueColumn() = default; + virtual T get_value(size_t idx) const = 0; +}; +template +class ValueColumn : public IValueColumn { + public: + ValueColumn() = default; + ~ValueColumn() = default; + + size_t size() const override { return data_.size(); } + + std::string column_info() const override { + return "ValueColumn<" + TypedConverter::name() + ">[" + + std::to_string(size()) + "]"; + } + ContextColumnType column_type() const override { + return ContextColumnType::kValue; + } + + std::shared_ptr builder() const override { + return std::make_shared>(); + } + + std::shared_ptr optional_builder() + const override { + return std::dynamic_pointer_cast( + std::make_shared>()); + } + std::shared_ptr dup() const override; + + std::shared_ptr shuffle( + const std::vector& offsets) const override; + + RTAnyType elem_type() const override { return TypedConverter::type(); } + RTAny get_elem(size_t idx) const override { + return TypedConverter::from_typed(data_[idx]); + } + + T get_value(size_t idx) const override { return data_[idx]; } + + ISigColumn* generate_signature() const override { + if constexpr (std::is_same_v or std::is_same_v) { + LOG(FATAL) << "not implemented for " << this->column_info(); + return nullptr; + } else { + return new SigColumn(data_); + } + } + + void generate_dedup_offset(std::vector& offsets) const override { + std::vector origin_offsets(data_.size()); + for (size_t i = 0; i < data_.size(); ++i) { + origin_offsets[i] = i; + } + std::sort(origin_offsets.begin(), origin_offsets.end(), + [this](size_t a, size_t b) { + // data_[a] == data_[b] + if (!(data_[a] < data_[b]) && !(data_[b] < data_[a])) { + return a < b; + } + return data_[a] < data_[b]; + }); + for (size_t i = 0; i < data_.size(); ++i) { + if (i == 0 || + ((data_[origin_offsets[i]] < data_[origin_offsets[i - 1]]) || + (data_[origin_offsets[i - 1]] < data_[origin_offsets[i]]))) { + offsets.push_back(origin_offsets[i]); + } + } + } + + std::shared_ptr union_col( + std::shared_ptr other) const override; + + private: + template + friend class ValueColumnBuilder; + std::vector data_; +}; + +template <> +class ValueColumn : public IValueColumn { + public: + ValueColumn() = default; + ~ValueColumn() = default; + + size_t size() const override { return data_.size(); } + + std::string column_info() const override { + return "ValueColumn<" + TypedConverter::name() + ">[" + + std::to_string(size()) + "]"; + } + ContextColumnType column_type() const override { + return ContextColumnType::kValue; + } + + std::shared_ptr builder() const override { + return std::dynamic_pointer_cast( + std::make_shared>()); + } + + RTAnyType elem_type() const override { return RTAnyType::kStringValue; } + std::shared_ptr shuffle( + const std::vector& offsets) const override; + std::shared_ptr dup() const override; + + RTAny get_elem(size_t idx) const override { + return RTAny::from_string(data_[idx]); + } + + std::string_view get_value(size_t idx) const override { + return std::string_view(data_[idx]); + } + + ISigColumn* generate_signature() const override { + return new SigColumn(data_); + } + + void generate_dedup_offset(std::vector& offsets) const override { + std::vector origin_offsets(data_.size()); + for (size_t i = 0; i < data_.size(); ++i) { + origin_offsets[i] = i; + } + std::sort(origin_offsets.begin(), origin_offsets.end(), + [this](size_t a, size_t b) { + if (data_[a] == data_[b]) { + return a < b; + } + return data_[a] < data_[b]; + }); + for (size_t i = 0; i < data_.size(); ++i) { + if (i == 0 || data_[origin_offsets[i]] != data_[origin_offsets[i - 1]]) { + offsets.push_back(origin_offsets[i]); + } + } + } + + private: + friend class ValueColumnBuilder; + std::vector data_; +}; + +template +class ValueColumnBuilder : public IContextColumnBuilder { + public: + ValueColumnBuilder() = default; + ~ValueColumnBuilder() = default; + + void reserve(size_t size) override { data_.reserve(size); } + void push_back_elem(const RTAny& val) override { + data_.push_back(TypedConverter::to_typed(val)); + } + + void push_back_opt(const T& val) { data_.push_back(val); } + + std::shared_ptr finish() override { + auto ret = std::make_shared>(); + ret->data_.swap(data_); + return ret; + } + + private: + std::vector data_; +}; + +template +class ListValueColumnBuilder; + +template +class ListValueColumn : public IValueColumn { + public: + ListValueColumn() = default; + ~ListValueColumn() = default; + + size_t size() const override { return data_.size(); } + + std::string column_info() const override { + return "ListValueColumn[" + std::to_string(size()) + "]"; + } + ContextColumnType column_type() const override { + return ContextColumnType::kValue; + } + + std::shared_ptr builder() const override { + auto builder = std::make_shared>(); + builder->set_list_impls(list_impls_); + builder->set_list_data(list_data_); + return builder; + } + + std::shared_ptr dup() const override; + std::shared_ptr shuffle( + const std::vector& offsets) const override; + RTAnyType elem_type() const override { + auto type = RTAnyType::kList; + return type; + } + RTAny get_elem(size_t idx) const override { + return RTAny::from_list(data_[idx]); + } + + List get_value(size_t idx) const override { return data_[idx]; } + + ISigColumn* generate_signature() const override { + LOG(FATAL) << "not implemented for " << this->column_info(); + return nullptr; + } + + void generate_dedup_offset(std::vector& offsets) const override { + std::vector origin_offsets(data_.size()); + for (size_t i = 0; i < data_.size(); ++i) { + origin_offsets[i] = i; + } + std::sort(origin_offsets.begin(), origin_offsets.end(), + [this](size_t a, size_t b) { + // data_[a] == data_[b] + if (!(data_[a] < data_[b]) && !(data_[b] < data_[a])) { + return a < b; + } + return data_[a] < data_[b]; + }); + for (size_t i = 0; i < data_.size(); ++i) { + if (i == 0 || + ((data_[origin_offsets[i]] < data_[origin_offsets[i - 1]]) || + (data_[origin_offsets[i - 1]] < data_[origin_offsets[i]]))) { + offsets.push_back(origin_offsets[i]); + } + } + } + + private: + template + friend class ListValueColumnBuilder; + std::vector data_; + std::vector> list_impls_; + std::vector> list_data_; +}; + +template +class ListValueColumnBuilder : public IContextColumnBuilder { + public: + ListValueColumnBuilder() = default; + ~ListValueColumnBuilder() = default; + + void reserve(size_t size) override { data_.reserve(size); } + void push_back_elem(const RTAny& val) override { + CHECK(val.type() == RTAnyType::kList); + data_.emplace_back(val.as_list()); + } + + void push_back_opt(const List& val) { data_.emplace_back(val); } + + void set_list_impls( + const std::vector>& list_impls) { + list_impls_ = list_impls; + } + + void set_list_data(const std::vector>& list_data) { + list_data_ = list_data; + } + + std::shared_ptr finish() override { + auto ret = std::make_shared>(); + ret->data_.swap(data_); + ret->list_impls_.swap(list_impls_); + ret->list_data_.swap(list_data_); + + return ret; + } + + private: + std::vector data_; + std::vector> list_impls_; + std::vector> list_data_; +}; +template +std::shared_ptr ListValueColumn::dup() const { + ListValueColumnBuilder builder; + for (const auto& list : data_) { + builder.push_back_opt(list); + } + builder.set_list_data(list_data_); + builder.set_list_impls(list_impls_); + return builder.finish(); +} + +template +std::shared_ptr ListValueColumn::shuffle( + const std::vector& offsets) const { + ListValueColumnBuilder builder; + builder.reserve(offsets.size()); + for (auto offset : offsets) { + builder.push_back_opt(data_[offset]); + } + builder.set_list_data(list_data_); + builder.set_list_impls(list_impls_); + return builder.finish(); +} + +class MapValueColumnBuilder; + +class MapValueColumn : public IValueColumn { + public: + MapValueColumn() = default; + ~MapValueColumn() = default; + + size_t size() const override { return values_.size(); } + + std::string column_info() const override { + return "MapValueColumn[" + std::to_string(size()) + "]"; + } + + ContextColumnType column_type() const override { + return ContextColumnType::kValue; + } + + std::shared_ptr builder() const override; + + std::shared_ptr dup() const override; + + std::shared_ptr shuffle( + const std::vector& offsets) const override; + + RTAnyType elem_type() const override { + auto type = RTAnyType::kMap; + return type; + } + + RTAny get_elem(size_t idx) const override { + auto map_impl = MapImpl::make_map_impl(&keys_, &values_[idx]); + auto map = Map::make_map(map_impl); + + return RTAny::from_map(map); + } + + Map get_value(size_t idx) const override { + auto map_impl = MapImpl::make_map_impl(&keys_, &values_[idx]); + return Map::make_map(map_impl); + } + + ISigColumn* generate_signature() const override { + LOG(FATAL) << "not implemented for " << this->column_info(); + return nullptr; + } + + void generate_dedup_offset(std::vector& offsets) const override { + LOG(FATAL) << "not implemented for " << this->column_info(); + } + + private: + friend class MapValueColumnBuilder; + std::vector keys_; + std::vector> values_; +}; + +class MapValueColumnBuilder : public IContextColumnBuilder { + public: + MapValueColumnBuilder() = default; + ~MapValueColumnBuilder() = default; + + void reserve(size_t size) override { values_.reserve(size); } + + void push_back_elem(const RTAny& val) override { + CHECK(val.type() == RTAnyType::kMap); + auto map = val.as_map(); + std::vector values; + const auto& [_, vals] = map.key_vals(); + for (const auto& v : *vals) { + values.push_back(v); + } + values_.push_back(std::move(values)); + } + + void push_back_opt(const std::vector& values) { + values_.push_back(values); + } + + void set_keys(const std::vector& keys) { keys_ = keys; } + + std::shared_ptr finish() override { + auto ret = std::make_shared(); + ret->keys_.swap(keys_); + ret->values_.swap(values_); + return ret; + } + + private: + friend class MapValueColumn; + std::vector keys_; + std::vector> values_; +}; + +template +class OptionalValueColumn : public IValueColumn { + public: + OptionalValueColumn() = default; + ~OptionalValueColumn() = default; + + size_t size() const override { return data_.size(); } + + std::string column_info() const override { + return "OptionalValueColumn<" + TypedConverter::name() + ">[" + + std::to_string(size()) + "]"; + } + ContextColumnType column_type() const override { + return ContextColumnType::kOptionalValue; + } + + std::shared_ptr dup() const override { + OptionalValueColumnBuilder builder; + for (size_t i = 0; i < data_.size(); ++i) { + builder.push_back_opt(data_[i], valid_[i]); + } + return builder.finish(); + } + + std::shared_ptr shuffle( + const std::vector& offsets) const override { + OptionalValueColumnBuilder builder; + builder.reserve(offsets.size()); + for (auto offset : offsets) { + builder.push_back_opt(data_[offset], valid_[offset]); + } + return builder.finish(); + } + + std::shared_ptr builder() const override { + return std::dynamic_pointer_cast( + std::make_shared>()); + } + + RTAnyType elem_type() const override { + auto type = TypedConverter::type(); + type.null_able_ = true; + return type; + } + RTAny get_elem(size_t idx) const override { + return TypedConverter::from_typed(data_[idx]); + } + + T get_value(size_t idx) const override { return data_[idx]; } + + ISigColumn* generate_signature() const override { + if constexpr (std::is_same_v or std::is_same_v) { + LOG(FATAL) << "not implemented for " << this->column_info(); + return nullptr; + } else { + return new SigColumn(data_); + } + } + + void generate_dedup_offset(std::vector& offsets) const override { + std::vector origin_offsets(data_.size()); + for (size_t i = 0; i < data_.size(); ++i) { + origin_offsets[i] = i; + } + std::sort(origin_offsets.begin(), origin_offsets.end(), + [this](size_t a, size_t b) { + // data_[a] == data_[b] + if (!(data_[a] < data_[b]) && !(data_[b] < data_[a])) { + return a < b; + } + return data_[a] < data_[b]; + }); + for (size_t i = 0; i < data_.size(); ++i) { + if (i == 0 || + ((data_[origin_offsets[i]] < data_[origin_offsets[i - 1]]) || + (data_[origin_offsets[i - 1]] < data_[origin_offsets[i]]))) { + offsets.push_back(origin_offsets[i]); + } + } + } + + bool has_value(size_t idx) const override { return valid_[idx]; } + bool is_optional() const override { return true; } + + private: + template + friend class OptionalValueColumnBuilder; + std::vector data_; + std::vector valid_; +}; + +template <> +class OptionalValueColumn + : public IValueColumn { + public: + OptionalValueColumn() = default; + ~OptionalValueColumn() = default; + + size_t size() const override { return data_.size(); } + + std::string column_info() const override { + return "OptionalValueColumn<" + TypedConverter::name() + + ">[" + std::to_string(size()) + "]"; + } + ContextColumnType column_type() const override { + return ContextColumnType::kOptionalValue; + } + + std::shared_ptr builder() const override { + return std::dynamic_pointer_cast( + std::make_shared>()); + } + + std::shared_ptr dup() const override; + + std::shared_ptr shuffle( + const std::vector& offsets) const override; + RTAnyType elem_type() const override { + auto type = RTAnyType::kStringValue; + type.null_able_ = true; + return type; + } + RTAny get_elem(size_t idx) const override { + return RTAny::from_string(data_[idx]); + } + + std::string_view get_value(size_t idx) const override { + return std::string_view(data_[idx]); + } + + ISigColumn* generate_signature() const override { + LOG(FATAL) << "not implemented for " << this->column_info(); + return nullptr; + } + + void generate_dedup_offset(std::vector& offsets) const override { + std::vector origin_offsets(data_.size()); + for (size_t i = 0; i < data_.size(); ++i) { + origin_offsets[i] = i; + } + std::sort(origin_offsets.begin(), origin_offsets.end(), + [this](size_t a, size_t b) { + if (data_[a] == data_[b]) { + return a < b; + } + return data_[a] < data_[b]; + }); + for (size_t i = 0; i < data_.size(); ++i) { + if (i == 0 || data_[origin_offsets[i]] != data_[origin_offsets[i - 1]]) { + offsets.push_back(origin_offsets[i]); + } + } + } + + bool has_value(size_t idx) const override { return valid_[idx]; } + bool is_optional() const override { return true; } + + private: + friend class OptionalValueColumnBuilder; + std::vector data_; + std::vector valid_; +}; + +template +class OptionalValueColumnBuilder : public IOptionalContextColumnBuilder { + public: + OptionalValueColumnBuilder() = default; + ~OptionalValueColumnBuilder() = default; + + void reserve(size_t size) override { + data_.reserve(size); + valid_.reserve(size); + } + + void push_back_elem(const RTAny& val) override { + data_.push_back(TypedConverter::to_typed(val)); + valid_.push_back(true); + } + + void push_back_opt(const T& val, bool valid) { + data_.push_back(val); + valid_.push_back(valid); + } + + void push_back_null() override { + data_.emplace_back(T()); + valid_.push_back(false); + } + + std::shared_ptr finish() override { + auto ret = std::make_shared>(); + ret->data_.swap(data_); + ret->valid_.swap(valid_); + return std::dynamic_pointer_cast(ret); + } + + private: + std::vector data_; + std::vector valid_; +}; + +template <> +class OptionalValueColumnBuilder + : public IOptionalContextColumnBuilder { + public: + OptionalValueColumnBuilder() = default; + ~OptionalValueColumnBuilder() = default; + + void reserve(size_t size) override { + data_.reserve(size); + valid_.reserve(size); + } + + void push_back_elem(const RTAny& val) override { + data_.push_back(std::string(val.as_string())); + valid_.push_back(true); + } + + void push_back_opt(const std::string& val, bool valid) { + data_.push_back(val); + valid_.push_back(valid); + } + + void push_back_null() override { + data_.emplace_back(); + valid_.push_back(false); + } + + std::shared_ptr finish() override { + auto ret = std::make_shared>(); + ret->data_.swap(data_); + ret->valid_.swap(valid_); + return std::dynamic_pointer_cast(ret); + } + + private: + std::vector data_; + std::vector valid_; +}; + +template <> +class ValueColumnBuilder : public IContextColumnBuilder { + public: + ValueColumnBuilder() = default; + ~ValueColumnBuilder() = default; + + void reserve(size_t size) override { data_.reserve(size); } + void push_back_elem(const RTAny& val) override { + data_.push_back(std::string(val.as_string())); + } + + void push_back_opt(const std::string& val) { data_.push_back(val); } + + std::shared_ptr finish() override { + auto ret = std::make_shared>(); + ret->data_.swap(data_); + return ret; + } + + private: + std::vector data_; +}; + +template +std::shared_ptr ValueColumn::shuffle( + const std::vector& offsets) const { + ValueColumnBuilder builder; + builder.reserve(offsets.size()); + for (auto offset : offsets) { + builder.push_back_opt(data_[offset]); + } + return builder.finish(); +} + +template +std::shared_ptr ValueColumn::dup() const { + ValueColumnBuilder builder; + for (auto v : data_) { + builder.push_back_opt(v); + } + return builder.finish(); +} + +template +std::shared_ptr ValueColumn::union_col( + std::shared_ptr other) const { + ValueColumnBuilder builder; + for (auto v : data_) { + builder.push_back_opt(v); + } + const ValueColumn& rhs = *std::dynamic_pointer_cast>(other); + for (auto v : rhs.data_) { + builder.push_back_opt(v); + } + return builder.finish(); +} + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_COMMON_COLUMNS_VALUE_COLUMNS_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/columns/vertex_columns.cc b/flex/engines/graph_db/runtime/common/columns/vertex_columns.cc new file mode 100644 index 000000000000..426e98974a40 --- /dev/null +++ b/flex/engines/graph_db/runtime/common/columns/vertex_columns.cc @@ -0,0 +1,249 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/common/columns/vertex_columns.h" + +namespace gs { +namespace runtime { + +std::shared_ptr SLVertexColumn::shuffle( + const std::vector& offsets) const { + SLVertexColumnBuilder builder(label_); + builder.reserve(offsets.size()); + for (auto offset : offsets) { + builder.push_back_opt(vertices_[offset]); + } + return builder.finish(); +} + +void SLVertexColumn::generate_dedup_offset(std::vector& offsets) const { + offsets.clear(); +#if 0 + std::set vset; + size_t n = vertices_.size(); + for (size_t i = 0; i != n; ++i) { + vid_t cur = vertices_[i]; + if (vset.find(cur) == vset.end()) { + offsets.push_back(i); + vset.insert(cur); + } + } +#else + std::vector bitset; + size_t vnum = vertices_.size(); + bitset.resize(vnum); + for (size_t i = 0; i < vnum; ++i) { + vid_t v = vertices_[i]; + if (bitset.size() <= v) { + bitset.resize(v + 1); + bitset[v] = true; + + offsets.push_back(i); + } else { + if (!bitset[v]) { + offsets.push_back(i); + bitset[v] = true; + } + } + } +#endif +} + +std::shared_ptr SLVertexColumn::union_col( + std::shared_ptr other) const { + CHECK(other->column_type() == ContextColumnType::kVertex); + const IVertexColumn& vertex_column = + *std::dynamic_pointer_cast(other); + if (vertex_column.vertex_column_type() == VertexColumnType::kSingle) { + const SLVertexColumn& col = + dynamic_cast(vertex_column); + if (label() == col.label()) { + SLVertexColumnBuilder builder(label()); + for (auto v : vertices_) { + builder.push_back_opt(v); + } + for (auto v : col.vertices_) { + builder.push_back_opt(v); + } + return builder.finish(); + } + } + LOG(FATAL) << "not support..."; + return nullptr; +} + +std::shared_ptr SLVertexColumn::dup() const { + SLVertexColumnBuilder builder(label_); + for (auto v : vertices_) { + builder.push_back_opt(v); + } + return builder.finish(); +} + +std::shared_ptr SLVertexColumnBuilder::finish() { + auto ret = std::make_shared(label_); + ret->vertices_.swap(vertices_); + return ret; +} + +std::shared_ptr MSVertexColumn::shuffle( + const std::vector& offsets) const { + MLVertexColumnBuilder builder; + builder.reserve(offsets.size()); + for (auto offset : offsets) { + builder.push_back_vertex(get_vertex(offset)); + } + return builder.finish(); +} + +std::shared_ptr MSVertexColumn::dup() const { + MSVertexColumnBuilder builder; + for (auto& pair : vertices_) { + for (auto v : pair.second) { + builder.push_back_vertex(std::make_pair(pair.first, v)); + } + } + return builder.finish(); +} + +ISigColumn* SLVertexColumn::generate_signature() const { + return new SigColumn(vertices_); +} + +ISigColumn* MSVertexColumn::generate_signature() const { + LOG(FATAL) << "not implemented..."; + return nullptr; +} + +std::shared_ptr MSVertexColumnBuilder::finish() { + if (!cur_list_.empty()) { + vertices_.emplace_back(cur_label_, std::move(cur_list_)); + cur_list_.clear(); + } + auto ret = std::make_shared(); + auto& label_set = ret->labels_; + for (auto& pair : vertices_) { + label_set.insert(pair.first); + } + ret->vertices_.swap(vertices_); + return ret; +} + +std::shared_ptr MLVertexColumn::shuffle( + const std::vector& offsets) const { + MLVertexColumnBuilder builder(labels_); + builder.reserve(offsets.size()); + for (auto offset : offsets) { + builder.push_back_vertex(vertices_[offset]); + } + return builder.finish(); +} + +ISigColumn* MLVertexColumn::generate_signature() const { + return new SigColumn>(vertices_); +} + +void MLVertexColumn::generate_dedup_offset(std::vector& offsets) const { + offsets.clear(); + std::set> vset; + size_t n = vertices_.size(); + for (size_t i = 0; i != n; ++i) { + auto cur = vertices_[i]; + if (vset.find(cur) == vset.end()) { + offsets.push_back(i); + vset.insert(cur); + } + } +} + +std::shared_ptr MLVertexColumn::dup() const { + MLVertexColumnBuilder builder(labels_); + for (auto& pair : vertices_) { + builder.push_back_vertex(pair); + } + return builder.finish(); +} + +std::shared_ptr MLVertexColumnBuilder::finish() { + auto ret = std::make_shared(); + ret->vertices_.swap(vertices_); + ret->labels_.swap(labels_); + return ret; +} + +void OptionalSLVertexColumn::generate_dedup_offset( + std::vector& offsets) const { + offsets.clear(); + std::vector bitset; + size_t vnum = vertices_.size(); + bitset.resize(vnum); + bool flag = false; + size_t idx = 0; + for (size_t i = 0; i < vnum; ++i) { + vid_t v = vertices_[i]; + if (v == std::numeric_limits::max()) { + if (!flag) { + flag = true; + idx = i; + continue; + } + } + if (bitset.size() <= v) { + bitset.resize(v + 1); + bitset[v] = true; + offsets.push_back(i); + } else { + if (!bitset[v]) { + offsets.push_back(i); + bitset[v] = true; + } + } + } + if (flag) { + offsets.push_back(idx); + } +} + +std::shared_ptr OptionalSLVertexColumn::dup() const { + OptionalSLVertexColumnBuilder builder(label_); + for (auto v : vertices_) { + builder.push_back_opt(v); + } + return builder.finish(); +} + +std::shared_ptr OptionalSLVertexColumn::shuffle( + const std::vector& offsets) const { + OptionalSLVertexColumnBuilder builder(label_); + builder.reserve(offsets.size()); + for (auto offset : offsets) { + builder.push_back_opt(vertices_[offset]); + } + return builder.finish(); +} + +ISigColumn* OptionalSLVertexColumn::generate_signature() const { + return new SigColumn(vertices_); +} + +std::shared_ptr OptionalSLVertexColumnBuilder::finish() { + auto ret = std::make_shared(label_); + ret->vertices_.swap(vertices_); + return ret; +} + +} // namespace runtime + +} // namespace gs diff --git a/flex/engines/graph_db/runtime/common/columns/vertex_columns.h b/flex/engines/graph_db/runtime/common/columns/vertex_columns.h new file mode 100644 index 000000000000..108ac48d5e5e --- /dev/null +++ b/flex/engines/graph_db/runtime/common/columns/vertex_columns.h @@ -0,0 +1,479 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RUNTIME_COMMON_COLUMNS_VERTEX_COLUMNS_H_ +#define RUNTIME_COMMON_COLUMNS_VERTEX_COLUMNS_H_ + +#include "flex/engines/graph_db/runtime/common/columns/i_context_column.h" + +namespace gs { + +namespace runtime { + +enum class VertexColumnType { + kSingle, + kMultiSegment, + kMultiple, + kSingleOptional, +}; + +class IVertexColumn : public IContextColumn { + public: + IVertexColumn() = default; + virtual ~IVertexColumn() = default; + + ContextColumnType column_type() const override { + return ContextColumnType::kVertex; + } + + virtual VertexColumnType vertex_column_type() const = 0; + virtual std::pair get_vertex(size_t idx) const = 0; + + RTAny get_elem(size_t idx) const override { + return RTAny::from_vertex(this->get_vertex(idx)); + } + + RTAnyType elem_type() const override { return RTAnyType::kVertex; } + + virtual std::set get_labels_set() const = 0; +}; + +class IVertexColumnBuilder : public IContextColumnBuilder { + public: + IVertexColumnBuilder() = default; + virtual ~IVertexColumnBuilder() = default; + + virtual void push_back_vertex(const std::pair& v) = 0; + + void push_back_elem(const RTAny& val) override { + this->push_back_vertex(val.as_vertex()); + } +}; + +class IOptionalVertexColumnBuilder : public IOptionalContextColumnBuilder { + public: + IOptionalVertexColumnBuilder() = default; + virtual ~IOptionalVertexColumnBuilder() = default; + + virtual void push_back_vertex(const std::pair& v) = 0; + + void push_back_elem(const RTAny& val) override { + this->push_back_vertex(val.as_vertex()); + } +}; + +class SLVertexColumnBuilder; +class OptionalSLVertexColumnBuilder; + +class SLVertexColumn : public IVertexColumn { + public: + SLVertexColumn(label_t label) : label_(label) {} + ~SLVertexColumn() = default; + + size_t size() const override { return vertices_.size(); } + + std::string column_info() const override { + return "SLVertexColumn(" + std::to_string(label_) + ")[" + + std::to_string(size()) + "]"; + } + + VertexColumnType vertex_column_type() const override { + return VertexColumnType::kSingle; + } + + std::shared_ptr shuffle( + const std::vector& offsets) const override; + + std::shared_ptr builder() const override { + auto ptr = std::make_shared(label_); + return std::dynamic_pointer_cast(ptr); + } + + std::shared_ptr optional_builder() + const override { + auto ptr = std::make_shared(label_); + return std::dynamic_pointer_cast(ptr); + } + + std::shared_ptr dup() const override; + + std::pair get_vertex(size_t idx) const override { + return std::make_pair(label_, vertices_[idx]); + } + + std::shared_ptr union_col( + std::shared_ptr other) const override; + + void generate_dedup_offset(std::vector& offsets) const override; + + template + void foreach_vertex(const FUNC_T& func) const { + size_t num = vertices_.size(); + for (size_t k = 0; k < num; ++k) { + func(k, label_, vertices_[k]); + } + } + + std::set get_labels_set() const override { + std::set ret; + ret.insert(label_); + return ret; + } + + label_t label() const { return label_; } + + ISigColumn* generate_signature() const override; + + private: + friend class SLVertexColumnBuilder; + std::vector vertices_; + label_t label_; +}; + +class SLVertexColumnBuilder : public IVertexColumnBuilder { + public: + SLVertexColumnBuilder(label_t label) : label_(label) {} + SLVertexColumnBuilder(const std::set& labels) + : label_(*labels.begin()) { + assert(labels.size() == 1); + } + ~SLVertexColumnBuilder() = default; + + void reserve(size_t size) override { vertices_.reserve(size); } + + void push_back_vertex(const std::pair& v) override { + assert(v.first == label_); + vertices_.push_back(v.second); + } + void push_back_opt(vid_t v) { vertices_.push_back(v); } + + std::shared_ptr finish() override; + + private: + std::vector vertices_; + label_t label_; +}; + +class OptionalSLVertexColumn : public IVertexColumn { + public: + OptionalSLVertexColumn(label_t label) : label_(label) {} + ~OptionalSLVertexColumn() = default; + + size_t size() const override { return vertices_.size(); } + + std::shared_ptr builder() const override { + auto ptr = std::make_shared(label_); + return std::dynamic_pointer_cast(ptr); + } + + std::string column_info() const override { + return "OptionalSLVertex[" + std::to_string(size()) + "]"; + } + + VertexColumnType vertex_column_type() const override { + return VertexColumnType::kSingleOptional; + } + + std::shared_ptr shuffle( + const std::vector& offsets) const override; + + std::shared_ptr dup() const override; + + std::pair get_vertex(size_t idx) const override { + return {label_, vertices_[idx]}; + } + + bool is_optional() const override { return true; } + + bool has_value(size_t idx) const override { + return vertices_[idx] != std::numeric_limits::max(); + } + + void generate_dedup_offset(std::vector& offsets) const override; + + template + void foreach_vertex(const FUNC_T& func) const { + size_t num = vertices_.size(); + for (size_t k = 0; k < num; ++k) { + func(k, label_, vertices_[k], 0); + } + } + + std::set get_labels_set() const override { return {label_}; } + + ISigColumn* generate_signature() const override; + + private: + friend class OptionalSLVertexColumnBuilder; + label_t label_; + std::vector vertices_; +}; + +class OptionalSLVertexColumnBuilder : public IOptionalVertexColumnBuilder { + public: + OptionalSLVertexColumnBuilder(label_t label) : label_(label) {} + ~OptionalSLVertexColumnBuilder() = default; + + void reserve(size_t size) override { vertices_.reserve(size); } + + void push_back_vertex(const std::pair& v) override { + vertices_.push_back(v.second); + } + + void push_back_opt(vid_t v) { vertices_.push_back(v); } + + void push_back_null() override { + vertices_.emplace_back(std::numeric_limits::max()); + } + + std::shared_ptr finish() override; + + private: + label_t label_; + std::vector vertices_; +}; + +class MSVertexColumnBuilder; +class MSVertexColumn : public IVertexColumn { + public: + MSVertexColumn() = default; + ~MSVertexColumn() = default; + + size_t size() const override { + size_t ret = 0; + for (auto& pair : vertices_) { + ret += pair.second.size(); + } + return ret; + } + + std::shared_ptr builder() const override { + auto ptr = std::make_shared(); + return std::dynamic_pointer_cast(ptr); + } + + std::string column_info() const override { + std::string labels; + for (auto label : labels_) { + labels += std::to_string(label); + labels += ", "; + } + if (!labels.empty()) { + labels.resize(labels.size() - 2); + } + return "MSVertexColumn(" + labels + ")[" + std::to_string(size()) + "]"; + } + + VertexColumnType vertex_column_type() const override { + return VertexColumnType::kMultiSegment; + } + + std::shared_ptr dup() const override; + + std::shared_ptr shuffle( + const std::vector& offsets) const override; + + std::pair get_vertex(size_t idx) const override { + for (auto& pair : vertices_) { + if (idx < pair.second.size()) { + return std::make_pair(pair.first, pair.second[idx]); + } + idx -= pair.second.size(); + } + LOG(FATAL) << "not found..."; + return std::make_pair(-1, std::numeric_limits::max()); + } + + template + void foreach_vertex(const FUNC_T& func) const { + size_t index = 0; + for (auto& pair : vertices_) { + label_t label = pair.first; + for (auto v : pair.second) { + func(index++, label, v); + } + } + } + + std::set get_labels_set() const override { return labels_; } + + ISigColumn* generate_signature() const override; + + private: + friend class MSVertexColumnBuilder; + std::vector>> vertices_; + std::set labels_; +}; + +#if 0 +class MSVertexColumnBuilder : public IVertexColumnBuilder { + public: + MSVertexColumnBuilder() = default; + ~MSVertexColumnBuilder() = default; + + void reserve(size_t size) override; + + void push_back_vertex(const std::pair& v) override; + + void start_label(label_t label); + + void push_back_opt(vid_t v); + + std::shared_ptr finish() override; + + private: + label_t cur_label_; + std::vector cur_list_; + + std::vector>> vertices_; +}; +#else +class MSVertexColumnBuilder : public IVertexColumnBuilder { + public: + MSVertexColumnBuilder() = default; + ~MSVertexColumnBuilder() = default; + + void reserve(size_t size) override {} + + void push_back_vertex(const std::pair& v) override { + if (v.first == cur_label_) { + cur_list_.push_back(v.second); + } else { + if (!cur_list_.empty()) { + vertices_.emplace_back(cur_label_, std::move(cur_list_)); + cur_list_.clear(); + } + cur_label_ = v.first; + cur_list_.push_back(v.second); + } + } + + void start_label(label_t label) { + if (!cur_list_.empty() && cur_label_ != label) { + vertices_.emplace_back(cur_label_, std::move(cur_list_)); + cur_list_.clear(); + } + cur_label_ = label; + } + + void push_back_opt(vid_t v) { cur_list_.push_back(v); } + + std::shared_ptr finish() override; + + private: + label_t cur_label_; + std::vector cur_list_; + + std::vector>> vertices_; +}; +#endif + +class MLVertexColumnBuilder; +class MLVertexColumn : public IVertexColumn { + public: + MLVertexColumn() = default; + ~MLVertexColumn() = default; + + size_t size() const override { return vertices_.size(); } + + std::shared_ptr builder() const override { + auto ptr = std::make_shared(labels_); + return std::dynamic_pointer_cast(ptr); + } + + std::string column_info() const override { + std::string labels; + for (auto label : labels_) { + labels += std::to_string(label); + labels += ", "; + } + if (!labels.empty()) { + labels.resize(labels.size() - 2); + } + return "MLVertexColumn(" + labels + ")[" + std::to_string(size()) + "]"; + } + + VertexColumnType vertex_column_type() const override { + return VertexColumnType::kMultiple; + } + + std::shared_ptr dup() const override; + + std::shared_ptr shuffle( + const std::vector& offsets) const override; + + std::pair get_vertex(size_t idx) const override { + return vertices_[idx]; + } + + template + void foreach_vertex(const FUNC_T& func) const { + size_t index = 0; + for (auto& pair : vertices_) { + func(index++, pair.first, pair.second); + } + } + + std::set get_labels_set() const override { return labels_; } + + ISigColumn* generate_signature() const override; + + void generate_dedup_offset(std::vector& offsets) const override; + + private: + friend class MLVertexColumnBuilder; + std::vector> vertices_; + std::set labels_; +}; + +class MLVertexColumnBuilder : public IVertexColumnBuilder { + public: + MLVertexColumnBuilder() = default; + MLVertexColumnBuilder(const std::set& labels) : labels_(labels) {} + ~MLVertexColumnBuilder() = default; + + void reserve(size_t size) override { vertices_.reserve(size); } + + void push_back_vertex(const std::pair& v) override { + labels_.insert(v.first); + vertices_.push_back(v); + } + + std::shared_ptr finish() override; + + private: + std::vector> vertices_; + std::set labels_; +}; + +template +void foreach_vertex(const IVertexColumn& col, const FUNC_T& func) { + if (col.vertex_column_type() == VertexColumnType::kSingle) { + const SLVertexColumn& ref = dynamic_cast(col); + ref.foreach_vertex(func); + } else if (col.vertex_column_type() == VertexColumnType::kMultiple) { + const MLVertexColumn& ref = dynamic_cast(col); + ref.foreach_vertex(func); + } else { + const MSVertexColumn& ref = dynamic_cast(col); + ref.foreach_vertex(func); + } +} + +} // namespace runtime + +} // namespace gs + +#endif // COMMON_COLUMNS_VERTEX_COLUMNS_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/context.cc b/flex/engines/graph_db/runtime/common/context.cc new file mode 100644 index 000000000000..65cb7050633a --- /dev/null +++ b/flex/engines/graph_db/runtime/common/context.cc @@ -0,0 +1,246 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/common/context.h" +#include "flex/engines/graph_db/runtime/common/columns/value_columns.h" +#include "flex/engines/graph_db/runtime/common/columns/vertex_columns.h" + +namespace gs { + +namespace runtime { + +Context::Context() : head(nullptr) {} + +void Context::clear() { + columns.clear(); + head.reset(); + idx_columns.clear(); + tag_ids.clear(); +} + +Context Context::dup() const { + Context new_ctx; + new_ctx.head = nullptr; + for (auto col : columns) { + if (col != nullptr) { + new_ctx.columns.push_back(col->dup()); + if (col == head) { + new_ctx.head = new_ctx.columns.back(); + } + } else { + new_ctx.columns.push_back(nullptr); + } + } + if (head != nullptr && new_ctx.head == nullptr) { + new_ctx.head = head->dup(); + } + for (auto& idx_col : idx_columns) { + new_ctx.idx_columns.emplace_back( + std::dynamic_pointer_cast>(idx_col->dup())); + } + new_ctx.tag_ids = tag_ids; + return new_ctx; +} + +void Context::update_tag_ids(const std::vector& tag_ids) { + this->tag_ids = tag_ids; +} + +void Context::append_tag_id(size_t tag_id) { + if (std::find(tag_ids.begin(), tag_ids.end(), tag_id) == tag_ids.end()) { + tag_ids.push_back(tag_id); + } +} + +void Context::set(int alias, std::shared_ptr col) { + head = col; + if (alias >= 0) { + if (columns.size() <= static_cast(alias)) { + columns.resize(alias + 1, nullptr); + } + assert(columns[alias] == nullptr); + columns[alias] = col; + } +} + +void Context::set_with_reshuffle(int alias, std::shared_ptr col, + const std::vector& offsets) { + head.reset(); + head = nullptr; + + if (alias >= 0) { + if (columns.size() > static_cast(alias) && + columns[alias] != nullptr) { + columns[alias].reset(); + columns[alias] = nullptr; + } + } + + reshuffle(offsets); + set(alias, col); +} + +void Context::set_with_reshuffle_beta(int alias, + std::shared_ptr col, + const std::vector& offsets, + const std::set& keep_cols) { + head.reset(); + head = nullptr; + if (alias >= 0) { + if (columns.size() > static_cast(alias) && + columns[alias] != nullptr) { + columns[alias].reset(); + columns[alias] = nullptr; + } + } + for (size_t k = 0; k < columns.size(); ++k) { + if (keep_cols.find(k) == keep_cols.end() && columns[k] != nullptr) { + columns[k].reset(); + columns[k] = nullptr; + } + } + + reshuffle(offsets); + + set(alias, col); +} + +void Context::reshuffle(const std::vector& offsets) { + bool head_shuffled = false; + std::vector> new_cols; + + for (auto col : columns) { + if (col == nullptr) { + new_cols.push_back(nullptr); + + continue; + } + if (col == head) { + head = col->shuffle(offsets); + new_cols.push_back(head); + head_shuffled = true; + } else { + new_cols.push_back(col->shuffle(offsets)); + } + } + if (!head_shuffled && head != nullptr) { + head = head->shuffle(offsets); + } + std::swap(new_cols, columns); + std::vector>> new_idx_columns; + for (auto& idx_col : idx_columns) { + new_idx_columns.emplace_back(std::dynamic_pointer_cast>( + idx_col->shuffle(offsets))); + } + std::swap(new_idx_columns, idx_columns); +} + +std::shared_ptr Context::get(int alias) { + assert(static_cast(alias) < columns.size()); + if (alias == -1) { + return head; + } else { + return columns[alias]; + } +} + +const std::shared_ptr Context::get(int alias) const { + assert(static_cast(alias) < columns.size()); + if (alias == -1) { + assert(head != nullptr); + return head; + } else { + assert(columns[alias] != nullptr); + return columns[alias]; + } +} + +size_t Context::row_num() const { + for (auto col : columns) { + if (col != nullptr) { + return col->size(); + } + } + if (head != nullptr) { + return head->size(); + } + return 0; +} + +void Context::desc(const std::string& info) const { + if (!info.empty()) { + LOG(INFO) << info; + } + for (size_t col_i = 0; col_i < col_num(); ++col_i) { + if (columns[col_i] != nullptr) { + LOG(INFO) << "\tcol-" << col_i << ": " << columns[col_i]->column_info(); + } + } + LOG(INFO) << "\thead: " << ((head == nullptr) ? "NULL" : head->column_info()); +} + +void Context::show(const ReadTransaction& txn) const { + size_t rn = row_num(); + size_t cn = col_num(); + for (size_t ri = 0; ri < rn; ++ri) { + std::string line; + for (size_t ci = 0; ci < cn; ++ci) { + if (columns[ci] != nullptr && + columns[ci]->column_type() == ContextColumnType::kVertex) { + auto v = std::dynamic_pointer_cast(columns[ci]) + ->get_vertex(ri); + int64_t id = txn.GetVertexId(v.first, v.second).AsInt64(); + line += std::to_string(id); + line += ", "; + } else if (columns[ci] != nullptr) { + line += columns[ci]->get_elem(ri).to_string(); + line += ", "; + } + } + LOG(INFO) << line; + } +} + +void Context::generate_idx_col(int idx) { + size_t n = row_num(); + ValueColumnBuilder builder; + builder.reserve(n); + for (size_t k = 0; k < n; ++k) { + builder.push_back_opt(k); + } + set(idx, builder.finish()); +} + +size_t Context::col_num() const { return columns.size(); } + +void Context::push_idx_col() { + ValueColumnBuilder builder; + builder.reserve(row_num()); + for (size_t k = 0; k < row_num(); ++k) { + builder.push_back_opt(k); + } + idx_columns.emplace_back( + std::dynamic_pointer_cast>(builder.finish())); +} + +const ValueColumn& Context::get_idx_col() const { + return *idx_columns.back(); +} + +void Context::pop_idx_col() { idx_columns.pop_back(); } + +} // namespace runtime + +} // namespace gs diff --git a/flex/engines/graph_db/runtime/common/context.h b/flex/engines/graph_db/runtime/common/context.h new file mode 100644 index 000000000000..ed32d93006a6 --- /dev/null +++ b/flex/engines/graph_db/runtime/common/context.h @@ -0,0 +1,80 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RUNTIME_COMMON_CONTEXT_H_ +#define RUNTIME_COMMON_CONTEXT_H_ + +#include "flex/engines/graph_db/database/read_transaction.h" +#include "flex/engines/graph_db/runtime/common/columns/i_context_column.h" +#include "flex/engines/graph_db/runtime/common/columns/value_columns.h" + +namespace gs { + +namespace runtime { + +class Context { + public: + Context(); + ~Context() = default; + + Context dup() const; + + void clear(); + + void update_tag_ids(const std::vector& tag_ids); + + void append_tag_id(size_t tag_id); + void set(int alias, std::shared_ptr col); + + void set_with_reshuffle(int alias, std::shared_ptr col, + const std::vector& offsets); + + void set_with_reshuffle_beta(int alias, std::shared_ptr col, + const std::vector& offsets, + const std::set& keep_cols); + + void reshuffle(const std::vector& offsets); + + std::shared_ptr get(int alias); + + const std::shared_ptr get(int alias) const; + + size_t row_num() const; + + void desc(const std::string& info = "") const; + + void show(const ReadTransaction& txn) const; + + void generate_idx_col(int idx); + + size_t col_num() const; + + void push_idx_col(); + + const ValueColumn& get_idx_col() const; + + void pop_idx_col(); + + std::vector> columns; + std::shared_ptr head; + std::vector>> idx_columns; + std::vector tag_ids; +}; + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_COMMON_CONTEXT_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/operators/dedup.cc b/flex/engines/graph_db/runtime/common/operators/dedup.cc new file mode 100644 index 000000000000..7caf2119c58d --- /dev/null +++ b/flex/engines/graph_db/runtime/common/operators/dedup.cc @@ -0,0 +1,128 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/common/operators/dedup.h" + +namespace gs { + +namespace runtime { + +void Dedup::dedup(const ReadTransaction& txn, Context& ctx, + const std::vector& cols) { + size_t row_num = ctx.row_num(); + std::vector offsets; + if (cols.size() == 0) { + return; + } else if (cols.size() == 1) { + ctx.get(cols[0])->generate_dedup_offset(offsets); + } else if (cols.size() == 2) { + ISigColumn* sig0 = ctx.get(cols[0])->generate_signature(); + ISigColumn* sig1 = ctx.get(cols[1])->generate_signature(); +#if 0 + std::set> sigset; + for (size_t r_i = 0; r_i < row_num; ++r_i) { + auto cur = std::make_pair(sig0->get_sig(r_i), sig1->get_sig(r_i)); + if (sigset.find(cur) == sigset.end()) { + offsets.push_back(r_i); + sigset.insert(cur); + } + } +#else + std::vector> list; + for (size_t r_i = 0; r_i < row_num; ++r_i) { + list.emplace_back(sig0->get_sig(r_i), sig1->get_sig(r_i), r_i); + } + std::sort(list.begin(), list.end()); + size_t list_size = list.size(); + if (list_size > 0) { + offsets.push_back(std::get<2>(list[0])); + for (size_t k = 1; k < list_size; ++k) { + if (std::get<0>(list[k]) != std::get<0>(list[k - 1]) || + std::get<1>(list[k]) != std::get<1>(list[k - 1])) { + offsets.push_back(std::get<2>(list[k])); + } + } + std::sort(offsets.begin(), offsets.end()); + } +#endif + delete sig0; + delete sig1; + } else if (cols.size() == 3) { + std::set> sigset; + ISigColumn* sig0 = ctx.get(cols[0])->generate_signature(); + ISigColumn* sig1 = ctx.get(cols[1])->generate_signature(); + ISigColumn* sig2 = ctx.get(cols[2])->generate_signature(); + for (size_t r_i = 0; r_i < row_num; ++r_i) { + auto cur = std::make_tuple(sig0->get_sig(r_i), sig1->get_sig(r_i), + sig2->get_sig(r_i)); + if (sigset.find(cur) == sigset.end()) { + offsets.push_back(r_i); + sigset.insert(cur); + } + } + delete sig0; + delete sig1; + delete sig2; + } else { + std::set set; + for (size_t r_i = 0; r_i < row_num; ++r_i) { + std::vector bytes; + Encoder encoder(bytes); + for (size_t c_i = 0; c_i < cols.size(); ++c_i) { + auto val = ctx.get(cols[c_i])->get_elem(r_i); + val.encode_sig(val.type(), encoder); + encoder.put_byte('#'); + } + std::string cur(bytes.begin(), bytes.end()); + if (set.find(cur) == set.end()) { + offsets.push_back(r_i); + set.insert(cur); + } + } + } + ctx.reshuffle(offsets); +} + +void Dedup::dedup(const ReadTransaction& txn, Context& ctx, + const std::vector& cols, + const std::vector>& vars) { + std::set set; + size_t row_num = ctx.row_num(); + std::vector offsets; + for (size_t r_i = 0; r_i < row_num; ++r_i) { + std::vector bytes; + Encoder encoder(bytes); + for (size_t c_i = 0; c_i < cols.size(); ++c_i) { + auto val = ctx.get(cols[c_i])->get_elem(r_i); + val.encode_sig(val.type(), encoder); + encoder.put_byte('#'); + } + for (auto& var : vars) { + auto val = var(r_i); + val.encode_sig(val.type(), encoder); + encoder.put_byte('#'); + } + std::string cur(bytes.begin(), bytes.end()); + if (set.find(cur) == set.end()) { + offsets.push_back(r_i); + set.insert(cur); + } + } + ctx.reshuffle(offsets); +} + +} // namespace runtime + +} // namespace gs diff --git a/flex/engines/graph_db/runtime/common/operators/dedup.h b/flex/engines/graph_db/runtime/common/operators/dedup.h new file mode 100644 index 000000000000..37810c701bfa --- /dev/null +++ b/flex/engines/graph_db/runtime/common/operators/dedup.h @@ -0,0 +1,43 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RUNTIME_COMMON_OPERATORS_DEDUP_H_ +#define RUNTIME_COMMON_OPERATORS_DEDUP_H_ + +#include +#include + +#include "flex/engines/graph_db/database/read_transaction.h" +#include "flex/engines/graph_db/runtime/common/context.h" + +namespace gs { + +namespace runtime { + + +class Dedup { + public: + static void dedup(const ReadTransaction& txn, Context& ctx, + const std::vector& cols); + static void dedup(const ReadTransaction& txn, Context& ctx, + const std::vector& cols, + const std::vector>& vars); +}; + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_COMMON_OPERATORS_DEDUP_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/operators/edge_expand.cc b/flex/engines/graph_db/runtime/common/operators/edge_expand.cc new file mode 100644 index 000000000000..3179069ca79f --- /dev/null +++ b/flex/engines/graph_db/runtime/common/operators/edge_expand.cc @@ -0,0 +1,822 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/common/operators/edge_expand.h" + +namespace gs { + +namespace runtime { + +static std::vector get_expand_label_set( + const ReadTransaction& txn, const std::set& label_set, + const std::vector& labels, Direction dir) { + std::vector label_triplets; + if (dir == Direction::kOut) { + for (auto& triplet : labels) { + if (label_set.count(triplet.src_label)) { + label_triplets.emplace_back(triplet); + } + } + } else if (dir == Direction::kIn) { + for (auto& triplet : labels) { + if (label_set.count(triplet.dst_label)) { + label_triplets.emplace_back(triplet); + } + } + } else { + for (auto& triplet : labels) { + if (label_set.count(triplet.src_label) || + label_set.count(triplet.dst_label)) { + label_triplets.emplace_back(triplet); + } + } + } + return label_triplets; +} + +Context EdgeExpand::expand_edge_without_predicate( + const ReadTransaction& txn, Context&& ctx, const EdgeExpandParams& params) { + std::vector shuffle_offset; + + if (params.labels.size() == 1) { + if (params.dir == Direction::kIn) { + auto& input_vertex_list = + *std::dynamic_pointer_cast(ctx.get(params.v_tag)); + label_t output_vertex_label = params.labels[0].src_label; + label_t edge_label = params.labels[0].edge_label; + + auto& props = txn.schema().get_edge_properties( + params.labels[0].src_label, params.labels[0].dst_label, + params.labels[0].edge_label); + PropertyType pt = PropertyType::kEmpty; + if (props.size() > 1) { + pt = PropertyType::kRecordView; + } else if (!props.empty()) { + pt = props[0]; + } + + SDSLEdgeColumnBuilder builder(Direction::kIn, params.labels[0], pt); + + label_t dst_label = params.labels[0].dst_label; + foreach_vertex(input_vertex_list, + [&](size_t index, label_t label, vid_t v) { + if (label != dst_label) { + return; + } + auto ie_iter = txn.GetInEdgeIterator( + label, v, output_vertex_label, edge_label); + while (ie_iter.IsValid()) { + auto nbr = ie_iter.GetNeighbor(); + CHECK(ie_iter.GetData().type == pt) + << ie_iter.GetData().type << " " << pt; + builder.push_back_opt(nbr, v, ie_iter.GetData()); + shuffle_offset.push_back(index); + ie_iter.Next(); + } + }); + + ctx.set_with_reshuffle(params.alias, builder.finish(), shuffle_offset); + return ctx; + } else if (params.dir == Direction::kOut) { + auto& input_vertex_list = + *std::dynamic_pointer_cast(ctx.get(params.v_tag)); + label_t output_vertex_label = params.labels[0].dst_label; + label_t edge_label = params.labels[0].edge_label; + + auto& props = txn.schema().get_edge_properties( + params.labels[0].src_label, params.labels[0].dst_label, + params.labels[0].edge_label); + PropertyType pt = PropertyType::kEmpty; + if (!props.empty()) { + pt = props[0]; + } + + SDSLEdgeColumnBuilder builder(Direction::kOut, params.labels[0], pt); + label_t src_label = params.labels[0].src_label; + foreach_vertex(input_vertex_list, + [&](size_t index, label_t label, vid_t v) { + if (label != src_label) { + return; + } + auto oe_iter = txn.GetOutEdgeIterator( + label, v, output_vertex_label, edge_label); + + while (oe_iter.IsValid()) { + auto nbr = oe_iter.GetNeighbor(); + CHECK(oe_iter.GetData().type == pt); + builder.push_back_opt(v, nbr, oe_iter.GetData()); + shuffle_offset.push_back(index); + oe_iter.Next(); + } + }); + + ctx.set_with_reshuffle(params.alias, builder.finish(), shuffle_offset); + return ctx; + } else { + auto& input_vertex_list = + *std::dynamic_pointer_cast(ctx.get(params.v_tag)); + auto props = txn.schema().get_edge_properties( + params.labels[0].src_label, params.labels[0].dst_label, + params.labels[0].edge_label); + PropertyType pt = PropertyType::kEmpty; + if (!props.empty()) { + pt = props[0]; + } + BDSLEdgeColumnBuilder builder(params.labels[0], pt); + foreach_vertex(input_vertex_list, [&](size_t index, label_t label, + vid_t v) { + if (label == params.labels[0].src_label) { + auto oe_iter = + txn.GetOutEdgeIterator(label, v, params.labels[0].dst_label, + params.labels[0].edge_label); + while (oe_iter.IsValid()) { + auto nbr = oe_iter.GetNeighbor(); + builder.push_back_opt(v, nbr, oe_iter.GetData(), Direction::kOut); + shuffle_offset.push_back(index); + oe_iter.Next(); + } + } + if (label == params.labels[0].dst_label) { + auto ie_iter = + txn.GetInEdgeIterator(label, v, params.labels[0].src_label, + params.labels[0].edge_label); + while (ie_iter.IsValid()) { + auto nbr = ie_iter.GetNeighbor(); + builder.push_back_opt(nbr, v, ie_iter.GetData(), Direction::kIn); + shuffle_offset.push_back(index); + ie_iter.Next(); + } + } + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), shuffle_offset); + return ctx; + } + } else { + auto column = + std::dynamic_pointer_cast(ctx.get(params.v_tag)); + auto label_set = column->get_labels_set(); + auto labels = + get_expand_label_set(txn, label_set, params.labels, params.dir); + std::vector> label_props; + for (auto& triplet : labels) { + auto& props = txn.schema().get_edge_properties( + triplet.src_label, triplet.dst_label, triplet.edge_label); + PropertyType pt = PropertyType::kEmpty; + if (!props.empty()) { + pt = props[0]; + } + label_props.emplace_back(triplet, pt); + } + if (params.dir == Direction::kOut || params.dir == Direction::kIn) { + if (labels.size() == 1) { + auto& input_vertex_list = + *std::dynamic_pointer_cast(ctx.get(params.v_tag)); + if (params.dir == Direction::kOut) { + auto& triplet = labels[0]; + SDSLEdgeColumnBuilder builder(Direction::kOut, triplet, + label_props[0].second); + foreach_vertex( + input_vertex_list, [&](size_t index, label_t label, vid_t v) { + if (label == triplet.src_label) { + auto oe_iter = txn.GetOutEdgeIterator( + label, v, triplet.dst_label, triplet.edge_label); + while (oe_iter.IsValid()) { + auto nbr = oe_iter.GetNeighbor(); + builder.push_back_opt(v, nbr, oe_iter.GetData()); + shuffle_offset.push_back(index); + oe_iter.Next(); + } + } + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), + shuffle_offset); + return ctx; + } else if (params.dir == Direction::kIn) { + auto& triplet = labels[0]; + SDSLEdgeColumnBuilder builder(Direction::kIn, triplet, + label_props[0].second); + foreach_vertex( + input_vertex_list, [&](size_t index, label_t label, vid_t v) { + if (label == triplet.dst_label) { + auto ie_iter = txn.GetInEdgeIterator( + label, v, triplet.src_label, triplet.edge_label); + while (ie_iter.IsValid()) { + auto nbr = ie_iter.GetNeighbor(); + builder.push_back_opt(nbr, v, ie_iter.GetData()); + shuffle_offset.push_back(index); + ie_iter.Next(); + } + } + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), + shuffle_offset); + return ctx; + } + } else if (labels.size() > 1 || labels.size() == 0) { + auto& input_vertex_list = + *std::dynamic_pointer_cast(ctx.get(params.v_tag)); + + SDMLEdgeColumnBuilder builder(params.dir, label_props); + if (params.dir == Direction::kOut) { + foreach_vertex(input_vertex_list, [&](size_t index, label_t label, + vid_t v) { + for (auto& triplet : labels) { + if (triplet.src_label == label) { + if (params.dir == Direction::kOut) { + auto oe_iter = txn.GetOutEdgeIterator( + label, v, triplet.dst_label, triplet.edge_label); + while (oe_iter.IsValid()) { + auto nbr = oe_iter.GetNeighbor(); + builder.push_back_opt(triplet, v, nbr, oe_iter.GetData()); + shuffle_offset.push_back(index); + oe_iter.Next(); + } + } + } + } + }); + } else { + foreach_vertex(input_vertex_list, [&](size_t index, label_t label, + vid_t v) { + for (auto& triplet : labels) { + if (triplet.dst_label == label) { + if (params.dir == Direction::kIn) { + auto ie_iter = txn.GetInEdgeIterator( + label, v, triplet.src_label, triplet.edge_label); + while (ie_iter.IsValid()) { + auto nbr = ie_iter.GetNeighbor(); + builder.push_back_opt(triplet, nbr, v, ie_iter.GetData()); + shuffle_offset.push_back(index); + ie_iter.Next(); + } + } + } + } + }); + } + + ctx.set_with_reshuffle(params.alias, builder.finish(), shuffle_offset); + return ctx; + } + } else if (params.dir == Direction::kBoth) { + if (labels.size() == 1) { + BDSLEdgeColumnBuilder builder(labels[0], label_props[0].second); + auto& input_vertex_list = + *std::dynamic_pointer_cast(ctx.get(params.v_tag)); + foreach_vertex(input_vertex_list, [&](size_t index, label_t label, + vid_t v) { + if (label == labels[0].src_label) { + auto oe_iter = txn.GetOutEdgeIterator(label, v, labels[0].dst_label, + labels[0].edge_label); + while (oe_iter.IsValid()) { + auto nbr = oe_iter.GetNeighbor(); + builder.push_back_opt(v, nbr, oe_iter.GetData(), Direction::kOut); + shuffle_offset.push_back(index); + oe_iter.Next(); + } + } + if (label == labels[0].dst_label) { + auto ie_iter = txn.GetInEdgeIterator(label, v, labels[0].src_label, + labels[0].edge_label); + while (ie_iter.IsValid()) { + auto nbr = ie_iter.GetNeighbor(); + builder.push_back_opt(nbr, v, ie_iter.GetData(), Direction::kIn); + shuffle_offset.push_back(index); + ie_iter.Next(); + } + } + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), shuffle_offset); + return ctx; + + } else { + BDMLEdgeColumnBuilder builder(label_props); + auto& input_vertex_list = + *std::dynamic_pointer_cast(ctx.get(params.v_tag)); + foreach_vertex( + input_vertex_list, [&](size_t index, label_t label, vid_t v) { + for (auto& triplet : labels) { + if (triplet.src_label == label) { + auto oe_iter = txn.GetOutEdgeIterator( + label, v, triplet.dst_label, triplet.edge_label); + while (oe_iter.IsValid()) { + auto nbr = oe_iter.GetNeighbor(); + builder.push_back_opt(triplet, v, nbr, oe_iter.GetData(), + Direction::kOut); + shuffle_offset.push_back(index); + oe_iter.Next(); + } + } + if (triplet.dst_label == label) { + auto ie_iter = txn.GetInEdgeIterator( + label, v, triplet.src_label, triplet.edge_label); + while (ie_iter.IsValid()) { + auto nbr = ie_iter.GetNeighbor(); + builder.push_back_opt(triplet, nbr, v, ie_iter.GetData(), + Direction::kIn); + shuffle_offset.push_back(index); + ie_iter.Next(); + } + } + } + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), shuffle_offset); + return ctx; + } + } + } + + LOG(FATAL) << "not support"; + return ctx; +} + +Context EdgeExpand::expand_vertex_without_predicate( + const ReadTransaction& txn, Context&& ctx, const EdgeExpandParams& params) { + std::shared_ptr input_vertex_list = + std::dynamic_pointer_cast(ctx.get(params.v_tag)); + VertexColumnType input_vertex_list_type = + input_vertex_list->vertex_column_type(); + + std::set output_vertex_set; + const std::set& input_vertex_set = + input_vertex_list->get_labels_set(); + if (params.dir == Direction::kOut) { + for (auto& triplet : params.labels) { + if (input_vertex_set.find(triplet.src_label) != input_vertex_set.end()) { + output_vertex_set.insert(triplet.dst_label); + } + } + } else if (params.dir == Direction::kIn) { + for (auto& triplet : params.labels) { + if (input_vertex_set.find(triplet.dst_label) != input_vertex_set.end()) { + output_vertex_set.insert(triplet.src_label); + } + } + } else { + for (auto& triplet : params.labels) { + if (input_vertex_set.find(triplet.src_label) != input_vertex_set.end()) { + output_vertex_set.insert(triplet.dst_label); + } + if (input_vertex_set.find(triplet.dst_label) != input_vertex_set.end()) { + output_vertex_set.insert(triplet.src_label); + } + } + } + + // if (output_vertex_set.empty()) { + // LOG(FATAL) << "output vertex label set is empty..."; + // } + + std::vector shuffle_offset; + + if (output_vertex_set.size() <= 1) { + label_t output_vertex_label = *output_vertex_set.begin(); + SLVertexColumnBuilder builder(output_vertex_label); + + if (input_vertex_list_type == VertexColumnType::kSingle) { + auto casted_input_vertex_list = + std::dynamic_pointer_cast(input_vertex_list); + label_t input_vertex_label = casted_input_vertex_list->label(); + if (params.labels.size() == 1) { + auto& label_triplet = params.labels[0]; + if (params.dir == Direction::kBoth && + label_triplet.src_label == label_triplet.dst_label && + label_triplet.src_label == output_vertex_label && + output_vertex_label == input_vertex_label) { + casted_input_vertex_list->foreach_vertex( + [&](size_t index, label_t label, vid_t v) { + auto oe_iter = txn.GetOutEdgeIterator(label, v, label, + label_triplet.edge_label); + while (oe_iter.IsValid()) { + auto nbr = oe_iter.GetNeighbor(); + builder.push_back_opt(nbr); + shuffle_offset.push_back(index); + oe_iter.Next(); + } + auto ie_iter = txn.GetInEdgeIterator(label, v, label, + label_triplet.edge_label); + while (ie_iter.IsValid()) { + auto nbr = ie_iter.GetNeighbor(); + builder.push_back_opt(nbr); + shuffle_offset.push_back(index); + ie_iter.Next(); + } + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), + shuffle_offset); + } else if (params.dir == Direction::kIn && + label_triplet.src_label == output_vertex_label && + label_triplet.dst_label == input_vertex_label) { + auto& props = txn.schema().get_edge_properties( + label_triplet.src_label, label_triplet.dst_label, + label_triplet.edge_label); + if (props.empty()) { + casted_input_vertex_list->foreach_vertex( + [&](size_t index, label_t label, vid_t v) { + auto iter = txn.GetInEdgeIterator( + label, v, output_vertex_label, label_triplet.edge_label); + while (iter.IsValid()) { + builder.push_back_opt(iter.GetNeighbor()); + shuffle_offset.push_back(index); + iter.Next(); + } + }); + } else if (props[0] == PropertyType::kDate) { + // also check csr mutability + const TypedMutableCsrBase* csr = + dynamic_cast*>( + txn.graph().get_ie_csr(label_triplet.dst_label, + label_triplet.src_label, + label_triplet.edge_label)); + casted_input_vertex_list->foreach_vertex( + [&](size_t index, label_t label, vid_t v) { + auto slice = csr->get_edges(v); + for (auto& e : slice) { + builder.push_back_opt(e.neighbor); + shuffle_offset.push_back(index); + } + }); + } else { + casted_input_vertex_list->foreach_vertex( + [&](size_t index, label_t label, vid_t v) { + auto ie_iter = txn.GetInEdgeIterator( + label, v, output_vertex_label, label_triplet.edge_label); + while (ie_iter.IsValid()) { + auto nbr = ie_iter.GetNeighbor(); + builder.push_back_opt(nbr); + shuffle_offset.push_back(index); + ie_iter.Next(); + } + }); + } + // casted_input_vertex_list->foreach_vertex( + // [&](size_t index, label_t label, vid_t v) { + // auto ie_iter = txn.GetInEdgeIterator( + // label, v, output_vertex_label, + // label_triplet.edge_label); + // while (ie_iter.IsValid()) { + // auto nbr = ie_iter.GetNeighbor(); + // builder.push_back_opt(nbr); + // shuffle_offset.push_back(index); + // ie_iter.Next(); + // } + // }); + ctx.set_with_reshuffle(params.alias, builder.finish(), + shuffle_offset); + } else if (params.dir == Direction::kOut && + label_triplet.src_label == input_vertex_label && + label_triplet.dst_label == output_vertex_label) { + casted_input_vertex_list->foreach_vertex( + [&](size_t index, label_t label, vid_t v) { + auto oe_iter = txn.GetOutEdgeIterator( + label, v, output_vertex_label, label_triplet.edge_label); + while (oe_iter.IsValid()) { + auto nbr = oe_iter.GetNeighbor(); + builder.push_back_opt(nbr); + shuffle_offset.push_back(index); + oe_iter.Next(); + } + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), + shuffle_offset); + } else { + LOG(FATAL) << "xxx"; + } + } else { + MLVertexColumnBuilder builder; + if (params.dir == Direction::kOut || params.dir == Direction::kIn) { + if (params.dir == Direction::kOut) { + casted_input_vertex_list->foreach_vertex( + [&](size_t index, label_t label, vid_t v) { + for (auto& triplet : params.labels) { + if (triplet.src_label == label) { + auto oe_iter = txn.GetOutEdgeIterator( + label, v, triplet.dst_label, triplet.edge_label); + while (oe_iter.IsValid()) { + auto nbr = oe_iter.GetNeighbor(); + builder.push_back_vertex( + std::make_pair(triplet.dst_label, nbr)); + shuffle_offset.push_back(index); + oe_iter.Next(); + } + } + } + }); + } else { + casted_input_vertex_list->foreach_vertex( + [&](size_t index, label_t label, vid_t v) { + for (auto& triplet : params.labels) { + if (triplet.dst_label == label) { + auto ie_iter = txn.GetInEdgeIterator( + label, v, triplet.src_label, triplet.edge_label); + while (ie_iter.IsValid()) { + auto nbr = ie_iter.GetNeighbor(); + builder.push_back_vertex( + std::make_pair(triplet.src_label, nbr)); + shuffle_offset.push_back(index); + ie_iter.Next(); + } + } + } + }); + } + ctx.set_with_reshuffle(params.alias, builder.finish(), + shuffle_offset); + } + } + } else if (input_vertex_list_type == VertexColumnType::kMultiple) { + auto casted_input_vertex_list = + std::dynamic_pointer_cast(input_vertex_list); + if (params.dir == Direction::kBoth) { + LOG(FATAL) << "AAAAAAAAA"; + } else if (params.dir == Direction::kIn) { + casted_input_vertex_list->foreach_vertex( + [&](size_t index, label_t label, vid_t v) { + for (auto& triplet : params.labels) { + if (triplet.dst_label == label) { + auto ie_iter = txn.GetInEdgeIterator( + label, v, triplet.src_label, triplet.edge_label); + while (ie_iter.IsValid()) { + auto nbr = ie_iter.GetNeighbor(); + builder.push_back_opt(nbr); + shuffle_offset.push_back(index); + ie_iter.Next(); + } + } + } + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), shuffle_offset); + } else if (params.dir == Direction::kOut) { + casted_input_vertex_list->foreach_vertex( + [&](size_t index, label_t label, vid_t v) { + for (auto& triplet : params.labels) { + if (triplet.src_label == label) { + auto oe_iter = txn.GetOutEdgeIterator( + label, v, triplet.dst_label, triplet.edge_label); + while (oe_iter.IsValid()) { + auto nbr = oe_iter.GetNeighbor(); + builder.push_back_opt(nbr); + shuffle_offset.push_back(index); + oe_iter.Next(); + } + } + } + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), shuffle_offset); + } else { + LOG(FATAL) << "xxx"; + } + } else if (input_vertex_list_type == VertexColumnType::kMultiSegment) { + auto casted_input_vertex_list = + std::dynamic_pointer_cast(input_vertex_list); + if (params.dir == Direction::kBoth) { + LOG(FATAL) << "AAAAAAAAA"; + } else if (params.dir == Direction::kIn) { + casted_input_vertex_list->foreach_vertex( + [&](size_t index, label_t label, vid_t v) { + for (auto& triplet : params.labels) { + if (triplet.dst_label == label) { + auto ie_iter = txn.GetInEdgeIterator( + label, v, triplet.src_label, triplet.edge_label); + while (ie_iter.IsValid()) { + auto nbr = ie_iter.GetNeighbor(); + builder.push_back_opt(nbr); + shuffle_offset.push_back(index); + ie_iter.Next(); + } + } + } + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), shuffle_offset); + } else if (params.dir == Direction::kOut) { + casted_input_vertex_list->foreach_vertex( + [&](size_t index, label_t label, vid_t v) { + for (auto& triplet : params.labels) { + if (triplet.src_label == label) { + auto oe_iter = txn.GetOutEdgeIterator( + label, v, triplet.dst_label, triplet.edge_label); + while (oe_iter.IsValid()) { + auto nbr = oe_iter.GetNeighbor(); + builder.push_back_opt(nbr); + shuffle_offset.push_back(index); + oe_iter.Next(); + } + } + } + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), shuffle_offset); + } else { + LOG(FATAL) << "xxx"; + } + } else { + LOG(FATAL) << "unexpected input vertex list type"; + } + } else { + if (input_vertex_list_type == VertexColumnType::kSingle) { + auto casted_input_vertex_list = + std::dynamic_pointer_cast(input_vertex_list); + label_t input_vertex_label = casted_input_vertex_list->label(); +#if 0 + MLVertexColumnBuilder builder; + for (label_t output_vertex_label : output_vertex_set) { + if (params.dir == Direction::kBoth) { + LOG(FATAL) << "AAAAA"; + } else if (params.dir == Direction::kIn) { + for (auto& triplet : params.labels) { + if (triplet.dst_label == input_vertex_label && + triplet.src_label == output_vertex_label) { + casted_input_vertex_list->foreach_vertex( + [&](size_t index, label_t label, vid_t v) { + auto ie_iter = txn.GetInEdgeIterator( + label, v, output_vertex_label, triplet.edge_label); + while (ie_iter.IsValid()) { + auto nbr = ie_iter.GetNeighbor(); + builder.push_back_vertex( + std::make_pair(output_vertex_label, nbr)); + shuffle_offset.push_back(index); + ie_iter.Next(); + } + }); + } + } + } else if (params.dir == Direction::kOut) { + LOG(FATAL) << "AAAAA"; + } + } +#else + MSVertexColumnBuilder builder; + for (label_t output_vertex_label : output_vertex_set) { + builder.start_label(output_vertex_label); + if (params.dir == Direction::kBoth) { + LOG(FATAL) << "AAAAA"; + } else if (params.dir == Direction::kIn) { + for (auto& triplet : params.labels) { + if (triplet.dst_label == input_vertex_label && + triplet.src_label == output_vertex_label) { + casted_input_vertex_list->foreach_vertex( + [&](size_t index, label_t label, vid_t v) { + auto ie_iter = txn.GetInEdgeIterator( + label, v, output_vertex_label, triplet.edge_label); + while (ie_iter.IsValid()) { + auto nbr = ie_iter.GetNeighbor(); + builder.push_back_opt(nbr); + shuffle_offset.push_back(index); + ie_iter.Next(); + } + }); + } + } + } else if (params.dir == Direction::kOut) { + LOG(FATAL) << "AAAAA"; + } + } +#endif + + ctx.set_with_reshuffle(params.alias, builder.finish(), shuffle_offset); + } else { + if (params.dir == Direction::kOut) { + auto& casted_input_vertex_list = + *std::dynamic_pointer_cast(input_vertex_list); + MLVertexColumnBuilder builder; + foreach_vertex(casted_input_vertex_list, [&](size_t index, + label_t label, vid_t v) { + for (auto& triplet : params.labels) { + if (triplet.src_label == label) { + auto oe_iter = txn.GetOutEdgeIterator(label, v, triplet.dst_label, + triplet.edge_label); + while (oe_iter.IsValid()) { + auto nbr = oe_iter.GetNeighbor(); + builder.push_back_vertex( + std::make_pair(triplet.dst_label, nbr)); + shuffle_offset.push_back(index); + oe_iter.Next(); + } + } + } + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), shuffle_offset); + return ctx; + } else if (params.dir == Direction::kBoth) { + auto& casted_input_vertex_list = + *std::dynamic_pointer_cast(input_vertex_list); + MLVertexColumnBuilder builder; + foreach_vertex(casted_input_vertex_list, [&](size_t index, + label_t label, vid_t v) { + for (auto& triplet : params.labels) { + if (triplet.src_label == label) { + auto oe_iter = txn.GetOutEdgeIterator(label, v, triplet.dst_label, + triplet.edge_label); + while (oe_iter.IsValid()) { + auto nbr = oe_iter.GetNeighbor(); + builder.push_back_vertex( + std::make_pair(triplet.dst_label, nbr)); + shuffle_offset.push_back(index); + oe_iter.Next(); + } + } + if (triplet.dst_label == label) { + auto ie_iter = txn.GetInEdgeIterator(label, v, triplet.src_label, + triplet.edge_label); + while (ie_iter.IsValid()) { + auto nbr = ie_iter.GetNeighbor(); + builder.push_back_vertex( + std::make_pair(triplet.src_label, nbr)); + shuffle_offset.push_back(index); + ie_iter.Next(); + } + } + } + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), shuffle_offset); + return ctx; + } else { + LOG(FATAL) << "not support"; + } + LOG(FATAL) << "edge expand vertex input multiple vertex label"; + } + } + + return ctx; +} + +Context EdgeExpand::expand_2d_vertex_without_predicate( + const ReadTransaction& txn, Context&& ctx, const EdgeExpandParams& params1, + const EdgeExpandParams& params2) { + std::shared_ptr input_vertex_list = + std::dynamic_pointer_cast(ctx.get(params1.v_tag)); + VertexColumnType input_vertex_list_type = + input_vertex_list->vertex_column_type(); + + std::vector shuffle_offset; + + if (params1.labels.size() == 1 && params2.labels.size() == 1) { + if (params1.dir == Direction::kOut && params2.dir == Direction::kOut) { + label_t d0_label = params1.labels[0].src_label; + label_t d1_label = params1.labels[0].dst_label; + label_t d2_label = params2.labels[0].dst_label; + label_t e0_label = params1.labels[0].edge_label; + label_t e1_label = params2.labels[0].edge_label; + + SLVertexColumnBuilder builder(d2_label); + + if (input_vertex_list_type == VertexColumnType::kSingle) { + auto casted_input_vertex_list = + std::dynamic_pointer_cast(input_vertex_list); + auto oe_csr0 = + txn.GetOutgoingSingleImmutableGraphView( + d0_label, d1_label, e0_label); + auto oe_csr1 = txn.GetOutgoingGraphView( + d1_label, d2_label, e1_label); + casted_input_vertex_list->foreach_vertex( + [&](size_t index, label_t label, vid_t v) { + if (oe_csr0.exist(v)) { + auto& oe0 = oe_csr0.get_edge(v); + auto oe1_slice = oe_csr1.get_edges(oe0.neighbor); + for (auto& e : oe1_slice) { + builder.push_back_opt(e.neighbor); + shuffle_offset.push_back(index); + } + } + + // auto oe_iter0 = + // txn.GetOutEdgeIterator(d0_label, v, d1_label, e0_label); + // while (oe_iter0.IsValid()) { + // auto nbr = oe_iter0.GetNeighbor(); + // auto oe_iter1 = + // txn.GetOutEdgeIterator(d1_label, nbr, d2_label, + // e1_label); + // while (oe_iter1.IsValid()) { + // auto nbr2 = oe_iter1.GetNeighbor(); + // builder.push_back_opt(nbr2); + // shuffle_offset.push_back(index); + // oe_iter1.Next(); + // } + // oe_iter0.Next(); + // } + }); + + ctx.set_with_reshuffle(params2.alias, builder.finish(), shuffle_offset); + return ctx; + } + } + } + LOG(FATAL) << "XXXX"; + + return ctx; +} + +} // namespace runtime + +} // namespace gs diff --git a/flex/engines/graph_db/runtime/common/operators/edge_expand.h b/flex/engines/graph_db/runtime/common/operators/edge_expand.h new file mode 100644 index 000000000000..90184a6ac50f --- /dev/null +++ b/flex/engines/graph_db/runtime/common/operators/edge_expand.h @@ -0,0 +1,394 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RUNTIME_COMMON_OPERATORS_EDGE_EXPAND_H_ +#define RUNTIME_COMMON_OPERATORS_EDGE_EXPAND_H_ + +#include + +#include "flex/engines/graph_db/database/read_transaction.h" +#include "flex/engines/graph_db/runtime/common/columns/edge_columns.h" +#include "flex/engines/graph_db/runtime/common/columns/vertex_columns.h" +#include "flex/engines/graph_db/runtime/common/context.h" + +#include "glog/logging.h" + +namespace gs { +namespace runtime { + +struct EdgeExpandParams { + int v_tag; + std::vector labels; + int alias; + Direction dir; +}; + +class EdgeExpand { + public: + template + static Context expand_edge(const ReadTransaction& txn, Context&& ctx, + const EdgeExpandParams& params, + const PRED_T& pred) { + std::vector shuffle_offset; + if (params.labels.size() == 1) { + if (params.dir == Direction::kIn) { + auto& input_vertex_list = + *std::dynamic_pointer_cast(ctx.get(params.v_tag)); + label_t output_vertex_label = params.labels[0].src_label; + label_t edge_label = params.labels[0].edge_label; + + auto& props = txn.schema().get_edge_properties( + params.labels[0].src_label, params.labels[0].dst_label, + params.labels[0].edge_label); + PropertyType pt = PropertyType::kEmpty; + if (!props.empty()) { + pt = props[0]; + } + + SDSLEdgeColumnBuilder builder(Direction::kIn, params.labels[0], pt); + + foreach_vertex(input_vertex_list, + [&](size_t index, label_t label, vid_t v) { + auto ie_iter = txn.GetInEdgeIterator( + label, v, output_vertex_label, edge_label); + while (ie_iter.IsValid()) { + auto nbr = ie_iter.GetNeighbor(); + if (pred(params.labels[0], nbr, v, ie_iter.GetData(), + Direction::kIn, index)) { + CHECK(ie_iter.GetData().type == pt); + builder.push_back_opt(nbr, v, ie_iter.GetData()); + shuffle_offset.push_back(index); + } + ie_iter.Next(); + } + }); + + ctx.set_with_reshuffle(params.alias, builder.finish(), shuffle_offset); + return ctx; + } else if (params.dir == Direction::kOut) { + auto& input_vertex_list = + *std::dynamic_pointer_cast(ctx.get(params.v_tag)); + label_t output_vertex_label = params.labels[0].dst_label; + label_t edge_label = params.labels[0].edge_label; + label_t src_label = params.labels[0].src_label; + + auto& props = txn.schema().get_edge_properties( + params.labels[0].src_label, params.labels[0].dst_label, + params.labels[0].edge_label); + PropertyType pt = PropertyType::kEmpty; + if (!props.empty()) { + pt = props[0]; + } + + SDSLEdgeColumnBuilder builder(Direction::kOut, params.labels[0], pt); + + foreach_vertex(input_vertex_list, + [&](size_t index, label_t label, vid_t v) { + if (label != src_label) { + return; + } + auto oe_iter = txn.GetOutEdgeIterator( + label, v, output_vertex_label, edge_label); + while (oe_iter.IsValid()) { + auto nbr = oe_iter.GetNeighbor(); + if (pred(params.labels[0], v, nbr, oe_iter.GetData(), + Direction::kOut, index)) { + CHECK(oe_iter.GetData().type == pt); + builder.push_back_opt(v, nbr, oe_iter.GetData()); + shuffle_offset.push_back(index); + } + oe_iter.Next(); + } + }); + + ctx.set_with_reshuffle(params.alias, builder.finish(), shuffle_offset); + return ctx; + } else { + LOG(FATAL) << "expand edge both"; + } + } else { + if (params.dir == Direction::kBoth) { + auto& input_vertex_list = + *std::dynamic_pointer_cast(ctx.get(params.v_tag)); + std::vector> label_props; + for (auto& triplet : params.labels) { + auto& props = txn.schema().get_edge_properties( + triplet.src_label, triplet.dst_label, triplet.edge_label); + PropertyType pt = PropertyType::kEmpty; + if (!props.empty()) { + pt = props[0]; + } + label_props.emplace_back(triplet, pt); + } + BDMLEdgeColumnBuilder builder(label_props); + + foreach_vertex( + input_vertex_list, [&](size_t index, label_t label, vid_t v) { + for (auto& label_prop : label_props) { + auto& triplet = label_prop.first; + auto& pt = label_prop.second; + if (label == triplet.src_label) { + auto oe_iter = txn.GetOutEdgeIterator( + label, v, triplet.dst_label, triplet.edge_label); + while (oe_iter.IsValid()) { + auto nbr = oe_iter.GetNeighbor(); + if (pred(triplet, v, nbr, oe_iter.GetData(), + Direction::kOut, index)) { + CHECK(oe_iter.GetData().type == pt); + builder.push_back_opt(triplet, v, nbr, oe_iter.GetData(), + Direction::kOut); + shuffle_offset.push_back(index); + } + oe_iter.Next(); + } + } + if (label == triplet.dst_label) { + auto ie_iter = txn.GetInEdgeIterator( + label, v, triplet.src_label, triplet.edge_label); + while (ie_iter.IsValid()) { + auto nbr = ie_iter.GetNeighbor(); + if (pred(triplet, nbr, v, ie_iter.GetData(), Direction::kIn, + index)) { + CHECK(ie_iter.GetData().type == pt); + builder.push_back_opt(triplet, nbr, v, ie_iter.GetData(), + Direction::kIn); + shuffle_offset.push_back(index); + } + ie_iter.Next(); + } + } + } + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), shuffle_offset); + return ctx; + } else if (params.dir == Direction::kOut) { + auto& input_vertex_list = + *std::dynamic_pointer_cast(ctx.get(params.v_tag)); + std::vector> label_props; + for (auto& triplet : params.labels) { + auto& props = txn.schema().get_edge_properties( + triplet.src_label, triplet.dst_label, triplet.edge_label); + PropertyType pt = PropertyType::kEmpty; + if (!props.empty()) { + pt = props[0]; + } + label_props.emplace_back(triplet, pt); + } + SDMLEdgeColumnBuilder builder(Direction::kOut, label_props); + + foreach_vertex( + input_vertex_list, [&](size_t index, label_t label, vid_t v) { + for (auto& label_prop : label_props) { + auto& triplet = label_prop.first; + auto& pt = label_prop.second; + if (label != triplet.src_label) + continue; + auto oe_iter = txn.GetOutEdgeIterator( + label, v, triplet.dst_label, triplet.edge_label); + while (oe_iter.IsValid()) { + auto nbr = oe_iter.GetNeighbor(); + if (pred(triplet, v, nbr, oe_iter.GetData(), Direction::kOut, + index)) { + CHECK(oe_iter.GetData().type == pt); + builder.push_back_opt(triplet, v, nbr, oe_iter.GetData()); + shuffle_offset.push_back(index); + } + oe_iter.Next(); + } + } + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), shuffle_offset); + return ctx; + } + } + LOG(FATAL) << "not support"; + } + + static Context expand_edge_without_predicate(const ReadTransaction& txn, + Context&& ctx, + const EdgeExpandParams& params); + + template + static Context expand_vertex(const ReadTransaction& txn, Context&& ctx, + const EdgeExpandParams& params, + const PRED_T& pred) { + std::shared_ptr input_vertex_list = + std::dynamic_pointer_cast(ctx.get(params.v_tag)); + VertexColumnType input_vertex_list_type = + input_vertex_list->vertex_column_type(); + + std::set output_vertex_set; + const std::set& input_vertex_set = + input_vertex_list->get_labels_set(); + if (params.dir == Direction::kOut) { + for (auto& triplet : params.labels) { + if (input_vertex_set.find(triplet.src_label) != + input_vertex_set.end()) { + output_vertex_set.insert(triplet.dst_label); + } + } + } else if (params.dir == Direction::kIn) { + for (auto& triplet : params.labels) { + if (input_vertex_set.find(triplet.dst_label) != + input_vertex_set.end()) { + output_vertex_set.insert(triplet.src_label); + } + } + } else { + for (auto& triplet : params.labels) { + if (input_vertex_set.find(triplet.src_label) != + input_vertex_set.end()) { + output_vertex_set.insert(triplet.dst_label); + } + if (input_vertex_set.find(triplet.dst_label) != + input_vertex_set.end()) { + output_vertex_set.insert(triplet.src_label); + } + } + } + + if (output_vertex_set.empty()) { + LOG(FATAL) << "output vertex label set is empty..."; + } + + std::vector shuffle_offset; + + if (output_vertex_set.size() == 1) { + label_t output_vertex_label = *output_vertex_set.begin(); + SLVertexColumnBuilder builder(output_vertex_label); + + if (input_vertex_list_type == VertexColumnType::kSingle) { + auto casted_input_vertex_list = + std::dynamic_pointer_cast(input_vertex_list); + label_t input_vertex_label = casted_input_vertex_list->label(); + if (params.labels.size() == 1) { + auto& label_triplet = params.labels[0]; + if (params.dir == Direction::kBoth && + label_triplet.src_label == label_triplet.dst_label && + label_triplet.src_label == output_vertex_label && + output_vertex_label == input_vertex_label) { + casted_input_vertex_list->foreach_vertex( + [&](size_t index, label_t label, vid_t v) { + auto oe_iter = txn.GetOutEdgeIterator( + label, v, label, label_triplet.edge_label); + while (oe_iter.IsValid()) { + auto nbr = oe_iter.GetNeighbor(); + if (pred(label_triplet, v, nbr, oe_iter.GetData(), + Direction::kOut, index)) { + builder.push_back_opt(nbr); + shuffle_offset.push_back(index); + } + oe_iter.Next(); + } + auto ie_iter = txn.GetInEdgeIterator( + label, v, label, label_triplet.edge_label); + while (ie_iter.IsValid()) { + auto nbr = ie_iter.GetNeighbor(); + if (pred(label_triplet, nbr, v, ie_iter.GetData(), + Direction::kIn, index)) { + builder.push_back_opt(nbr); + shuffle_offset.push_back(index); + } + ie_iter.Next(); + } + }); + + ctx.set_with_reshuffle(params.alias, builder.finish(), + shuffle_offset); + } else if (params.dir == Direction::kIn && + label_triplet.src_label == output_vertex_label && + label_triplet.dst_label == input_vertex_label) { + casted_input_vertex_list->foreach_vertex( + [&](size_t index, label_t label, vid_t v) { + auto ie_iter = txn.GetInEdgeIterator( + label, v, output_vertex_label, label_triplet.edge_label); + while (ie_iter.IsValid()) { + auto nbr = ie_iter.GetNeighbor(); + if (pred(label_triplet, nbr, v, ie_iter.GetData(), + Direction::kIn, index)) { + builder.push_back_opt(nbr); + shuffle_offset.push_back(index); + } + ie_iter.Next(); + } + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), + shuffle_offset); + } else { + LOG(FATAL) << "xxx, " << (int) params.dir; + } + } else { + LOG(FATAL) << "multiple label triplet..."; + } + } else { + LOG(FATAL) << "edge expand vertex input multiple vertex label"; + } + } else { + MLVertexColumnBuilder builder; + + if (input_vertex_list_type == VertexColumnType::kSingle) { + auto casted_input_vertex_list = + std::dynamic_pointer_cast(input_vertex_list); + label_t input_vertex_label = casted_input_vertex_list->label(); + for (label_t output_vertex_label : output_vertex_set) { + if (params.dir == Direction::kBoth) { + LOG(FATAL) << "AAAAA"; + } else if (params.dir == Direction::kIn) { + for (auto& triplet : params.labels) { + if (triplet.dst_label == input_vertex_label && + triplet.src_label == output_vertex_label) { + casted_input_vertex_list->foreach_vertex( + [&](size_t index, label_t label, vid_t v) { + auto ie_iter = txn.GetInEdgeIterator( + label, v, output_vertex_label, triplet.edge_label); + while (ie_iter.IsValid()) { + auto nbr = ie_iter.GetNeighbor(); + if (pred(triplet, nbr, v, ie_iter.GetData(), + Direction::kIn, index)) { + builder.push_back_vertex( + std::make_pair(output_vertex_label, nbr)); + shuffle_offset.push_back(index); + } + ie_iter.Next(); + } + }); + } + } + } else if (params.dir == Direction::kOut) { + LOG(FATAL) << "AAAAA"; + } + } + ctx.set_with_reshuffle(params.alias, builder.finish(), shuffle_offset); + } else { + LOG(FATAL) << "edge expand vertex input multiple vertex label"; + } + } + + return ctx; + } + + static Context expand_vertex_without_predicate( + const ReadTransaction& txn, Context&& ctx, + const EdgeExpandParams& params); + + static Context expand_2d_vertex_without_predicate( + const ReadTransaction& txn, Context&& ctx, + const EdgeExpandParams& params1, const EdgeExpandParams& params2); +}; + +} // namespace runtime +} // namespace gs + +#endif // RUNTIME_COMMON_OPERATORS_EDGE_EXPAND_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/operators/get_v.h b/flex/engines/graph_db/runtime/common/operators/get_v.h new file mode 100644 index 000000000000..499616b57048 --- /dev/null +++ b/flex/engines/graph_db/runtime/common/operators/get_v.h @@ -0,0 +1,394 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RUNTIME_COMMON_OPERATORS_GET_V_H_ +#define RUNTIME_COMMON_OPERATORS_GET_V_H_ + +#include "flex/engines/graph_db/runtime/common/columns/edge_columns.h" +#include "flex/engines/graph_db/runtime/common/columns/path_columns.h" +#include "flex/engines/graph_db/runtime/common/columns/vertex_columns.h" +#include "flex/engines/graph_db/runtime/common/context.h" + +namespace gs { +namespace runtime { + +struct GetVParams { + VOpt opt; + int tag; + std::vector tables; + int alias; +}; + +std::vector extract_labels(const std::vector& labels, + const std::vector& tables, + VOpt opt) { + std::vector output_labels; + for (const auto& label : labels) { + if (opt == VOpt::kStart) { + if (std::find(tables.begin(), tables.end(), label.src_label) != + tables.end() || + tables.size() == 0) { + output_labels.push_back(label.src_label); + } + } else if (opt == VOpt::kEnd) { + if (std::find(tables.begin(), tables.end(), label.dst_label) != + tables.end() || + tables.size() == 0) { + output_labels.push_back(label.dst_label); + } + } else { + LOG(FATAL) << "not support" << static_cast(opt); + } + } + return output_labels; +} +class GetV { + public: + template + static Context get_vertex_from_edges(const ReadTransaction& txn, + Context&& ctx, const GetVParams& params, + const PRED_T& pred) { + std::vector shuffle_offset; + auto col = ctx.get(params.tag); + if (col->column_type() == ContextColumnType::kPath) { + auto& input_path_list = + *std::dynamic_pointer_cast(col); + + MLVertexColumnBuilder builder; + input_path_list.foreach_path([&](size_t index, const Path& path) { + auto [label, vid] = path.get_end(); + builder.push_back_vertex(std::make_pair(label, vid)); + shuffle_offset.push_back(index); + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), shuffle_offset); + return ctx; + } + + auto column = std::dynamic_pointer_cast(ctx.get(params.tag)); + CHECK(column != nullptr); + if (column->edge_column_type() == EdgeColumnType::kSDSL) { + auto& input_edge_list = + *std::dynamic_pointer_cast(column); + // label_t output_vertex_label = params.tables[0]; + label_t output_vertex_label{0}; + auto edge_label = input_edge_list.get_labels()[0]; + + VOpt opt = params.opt; + if (params.opt == VOpt::kOther) { + if (input_edge_list.dir() == Direction::kOut) { + opt = VOpt::kEnd; + } else { + opt = VOpt::kStart; + } + } + if (opt == VOpt::kStart) { + output_vertex_label = edge_label.src_label; + } else if (opt == VOpt::kEnd) { + output_vertex_label = edge_label.dst_label; + } else { + LOG(FATAL) << "not support"; + } + // params tables size may be 0 + if (params.tables.size() == 1) { + CHECK(output_vertex_label == params.tables[0]); + } + SLVertexColumnBuilder builder(output_vertex_label); + if (opt == VOpt::kStart) { + input_edge_list.foreach_edge( + [&](size_t index, const LabelTriplet& label, vid_t src, vid_t dst, + const Any& edata, Direction dir) { + if (pred(label.src_label, src, index)) { + builder.push_back_opt(src); + shuffle_offset.push_back(index); + } + }); + } else if (opt == VOpt::kEnd) { + input_edge_list.foreach_edge( + [&](size_t index, const LabelTriplet& label, vid_t src, vid_t dst, + const Any& edata, Direction dir) { + if (pred(label.dst_label, dst, index)) { + builder.push_back_opt(dst); + shuffle_offset.push_back(index); + } + }); + } else { + LOG(FATAL) << "not support"; + } + ctx.set_with_reshuffle(params.alias, builder.finish(), shuffle_offset); + return ctx; + } else if (column->edge_column_type() == EdgeColumnType::kSDML) { + auto& input_edge_list = + *std::dynamic_pointer_cast(column); + VOpt opt = params.opt; + if (params.opt == VOpt::kOther) { + if (input_edge_list.dir() == Direction::kOut) { + opt = VOpt::kEnd; + } else { + opt = VOpt::kStart; + } + } + + auto labels = + extract_labels(input_edge_list.get_labels(), params.tables, opt); + if (labels.size() == 0) { + return ctx; + } + if (labels.size() > 1) { + MLVertexColumnBuilder builder; + if (opt == VOpt::kStart) { + input_edge_list.foreach_edge([&](size_t index, + const LabelTriplet& label, vid_t src, + vid_t dst, const Any& edata, + Direction dir) { + if (std::find(labels.begin(), labels.end(), label.src_label) != + labels.end()) { + builder.push_back_vertex(std::make_pair(label.src_label, src)); + shuffle_offset.push_back(index); + } + }); + } else if (opt == VOpt::kEnd) { + input_edge_list.foreach_edge([&](size_t index, + const LabelTriplet& label, vid_t src, + vid_t dst, const Any& edata, + Direction dir) { + if (std::find(labels.begin(), labels.end(), label.dst_label) != + labels.end()) { + builder.push_back_vertex(std::make_pair(label.dst_label, dst)); + shuffle_offset.push_back(index); + } + }); + } else { + LOG(FATAL) << "not support"; + } + ctx.set_with_reshuffle(params.alias, builder.finish(), shuffle_offset); + return ctx; + } + } else if (column->edge_column_type() == EdgeColumnType::kBDSL) { + auto& input_edge_list = + *std::dynamic_pointer_cast(column); + if (params.tables.size() == 0) { + auto type = input_edge_list.get_labels()[0]; + if (type.src_label != type.dst_label) { + MLVertexColumnBuilder builder; + CHECK(params.opt == VOpt::kOther); + input_edge_list.foreach_edge([&](size_t index, + const LabelTriplet& label, vid_t src, + vid_t dst, const Any& edata, + Direction dir) { + if (dir == Direction::kOut) { + builder.push_back_vertex(std::make_pair(label.dst_label, dst)); + } else { + builder.push_back_vertex(std::make_pair(label.src_label, src)); + } + shuffle_offset.push_back(index); + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), + shuffle_offset); + return ctx; + } else { + SLVertexColumnBuilder builder(type.src_label); + input_edge_list.foreach_edge( + [&](size_t index, const LabelTriplet& label, vid_t src, vid_t dst, + const Any& edata, Direction dir) { + if (dir == Direction::kOut) { + builder.push_back_opt(dst); + shuffle_offset.push_back(index); + } else { + builder.push_back_opt(src); + shuffle_offset.push_back(index); + } + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), + shuffle_offset); + return ctx; + } + } else { + std::vector labels; + auto type = input_edge_list.get_labels()[0]; + for (auto& label : params.tables) { + if (label == type.src_label || label == type.dst_label) { + labels.push_back(label); + } + } + if (labels.size() == 1) { + SLVertexColumnBuilder builder(labels[0]); + input_edge_list.foreach_edge( + [&](size_t index, const LabelTriplet& label, vid_t src, vid_t dst, + const Any& edata, Direction dir) { + if (dir == Direction::kOut) { + if (label.dst_label == labels[0]) { + builder.push_back_opt(dst); + shuffle_offset.push_back(index); + } + } else { + if (label.src_label == labels[0]) { + builder.push_back_opt(src); + shuffle_offset.push_back(index); + } + } + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), + shuffle_offset); + return ctx; + } else { + MLVertexColumnBuilder builder; + input_edge_list.foreach_edge([&](size_t index, + const LabelTriplet& label, vid_t src, + vid_t dst, const Any& edata, + Direction dir) { + if (dir == Direction::kOut) { + if (std::find(labels.begin(), labels.end(), label.dst_label) != + labels.end()) { + builder.push_back_vertex(std::make_pair(label.dst_label, dst)); + shuffle_offset.push_back(index); + } + } else { + if (std::find(labels.begin(), labels.end(), label.src_label) != + labels.end()) { + builder.push_back_vertex(std::make_pair(label.src_label, src)); + shuffle_offset.push_back(index); + } + } + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), + shuffle_offset); + return ctx; + } + } + } else if (column->edge_column_type() == EdgeColumnType::kBDML) { + auto& input_edge_list = + *std::dynamic_pointer_cast(column); + if (params.tables.size() == 0) { + MLVertexColumnBuilder builder; + CHECK(params.opt == VOpt::kOther); + input_edge_list.foreach_edge( + [&](size_t index, const LabelTriplet& label, vid_t src, vid_t dst, + const Any& edata, Direction dir) { + if (dir == Direction::kOut) { + builder.push_back_vertex(std::make_pair(label.dst_label, dst)); + } else { + builder.push_back_vertex(std::make_pair(label.src_label, src)); + } + shuffle_offset.push_back(index); + }); + + ctx.set_with_reshuffle(params.alias, builder.finish(), shuffle_offset); + return ctx; + } else { + if (params.tables.size() == 1) { + auto vlabel = params.tables[0]; + SLVertexColumnBuilder builder(vlabel); + input_edge_list.foreach_edge( + [&](size_t index, const LabelTriplet& label, vid_t src, vid_t dst, + const Any& edata, Direction dir) { + if (dir == Direction::kOut) { + if (label.dst_label == vlabel) { + builder.push_back_opt(dst); + shuffle_offset.push_back(index); + } + } else { + if (label.src_label == vlabel) { + builder.push_back_opt(src); + shuffle_offset.push_back(index); + } + } + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), + shuffle_offset); + + } else { + std::vector labels(std::numeric_limits::max(), false); + for (auto& label : params.tables) { + labels[label] = true; + } + MLVertexColumnBuilder builder; + input_edge_list.foreach_edge([&](size_t index, + const LabelTriplet& label, vid_t src, + vid_t dst, const Any& edata, + Direction dir) { + if (dir == Direction::kOut) { + if (labels[label.dst_label]) { + builder.push_back_vertex(std::make_pair(label.dst_label, dst)); + shuffle_offset.push_back(index); + } + } else { + if (labels[label.src_label]) { + builder.push_back_vertex(std::make_pair(label.src_label, src)); + shuffle_offset.push_back(index); + } + } + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), + shuffle_offset); + } + return ctx; + } + } + + LOG(FATAL) << "not support" << static_cast(column->edge_column_type()); + return ctx; + } + + template + static Context get_vertex_from_vertices(const ReadTransaction& txn, + Context&& ctx, + const GetVParams& params, + const PRED_T& pred) { + std::shared_ptr input_vertex_list_ptr = + std::dynamic_pointer_cast(ctx.get(params.tag)); + const IVertexColumn& input_vertex_list = *input_vertex_list_ptr; + + std::vector offset; + if (params.tag == params.alias) { + foreach_vertex(input_vertex_list, + [&](size_t idx, label_t label, vid_t v) { + if (pred(label, v, idx)) { + offset.push_back(idx); + } + }); + ctx.reshuffle(offset); + } else { + const std::set& label_set = input_vertex_list.get_labels_set(); + if (label_set.size() == 1) { + SLVertexColumnBuilder builder(*label_set.begin()); + foreach_vertex(input_vertex_list, + [&](size_t idx, label_t label, vid_t v) { + if (pred(label, v, idx)) { + builder.push_back_opt(v); + offset.push_back(idx); + } + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), offset); + + } else { + MLVertexColumnBuilder builder; + foreach_vertex(input_vertex_list, + [&](size_t idx, label_t label, vid_t v) { + if (pred(label, v, idx)) { + builder.push_back_vertex(std::make_pair(label, v)); + offset.push_back(idx); + } + }); + ctx.set_with_reshuffle(params.alias, builder.finish(), offset); + } + } + return ctx; + } +}; + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_COMMON_OPERATORS_GET_V_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/operators/group_by.h b/flex/engines/graph_db/runtime/common/operators/group_by.h new file mode 100644 index 000000000000..4c9d681f339f --- /dev/null +++ b/flex/engines/graph_db/runtime/common/operators/group_by.h @@ -0,0 +1,178 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RUNTIME_COMMON_OPERATORS_GROUP_BY_H_ +#define RUNTIME_COMMON_OPERATORS_GROUP_BY_H_ + +#include +#include +#include + +#include "flex/engines/graph_db/runtime/common/columns/value_columns.h" +#include "flex/engines/graph_db/runtime/common/context.h" +#include "flex/engines/graph_db/runtime/common/operators/project.h" + +namespace gs { + +namespace runtime { + +void aggregate_value_impl( + const std::tuple<>&, const std::vector>& to_aggregate, + std::vector>& output) {} + +template +void aggregate_value_impl( + const std::tuple, Rest...>& t, + const std::vector>& to_aggregate, + std::vector>& output) { + const ProjectExpr& cur = std::get<0>(t); + using ELEM_T = typename T::elem_t; + + ValueColumnBuilder builder; + builder.reserve(to_aggregate.size()); + for (size_t k = 0; k < to_aggregate.size(); ++k) { + builder.push_back_opt(cur.expr.reduce(to_aggregate[k])); + } + + if (output.size() <= cur.alias) { + output.resize(cur.alias + 1, nullptr); + } + output[cur.alias] = builder.finish(); + + aggregate_value_impl(tail(t), to_aggregate, output); +} + +class GroupBy { + public: + template + static Context group_by(Context&& ctx, const std::vector& keys, + const std::tuple& funcs) { + size_t row_num = ctx.row_num(); + std::vector offsets; + std::vector> to_aggregate; + + if (keys.size() == 0) { + return ctx; + } else if (keys.size() == 1) { + ISigColumn* sig = ctx.get(keys[0])->generate_signature(); +#if 1 + std::unordered_map sig_to_root; + for (size_t r_i = 0; r_i < row_num; ++r_i) { + size_t cur = sig->get_sig(r_i); + auto iter = sig_to_root.find(cur); + if (iter == sig_to_root.end()) { + sig_to_root.emplace(cur, offsets.size()); + offsets.push_back(r_i); + std::vector list; + list.push_back(r_i); + to_aggregate.emplace_back(std::move(list)); + } else { + to_aggregate[iter->second].push_back(r_i); + } + } +#else + std::vector> vec; + vec.reserve(row_num); + for (size_t r_i = 0; r_i < row_num; ++r_i) { + size_t cur = sig->get_sig(r_i); + vec.emplace_back(cur, r_i); + } + std::sort(vec.begin(), vec.end()); + if (row_num > 0) { + std::vector ta; + size_t cur = vec[0].first; + ta.push_back(vec[0].second); + offsets.push_back(vec[0].second); + for (size_t k = 1; k < row_num; ++k) { + if (vec[k].first != cur) { + to_aggregate.emplace_back(std::move(ta)); + ta.clear(); + cur = vec[k].first; + ta.push_back(vec[k].second); + offsets.push_back(vec[k].second); + } else { + ta.push_back(vec[k].second); + } + } + if (!ta.empty()) { + to_aggregate.emplace_back(std::move(ta)); + } + } +#endif + delete sig; + } else if (keys.size() == 2) { + std::map, size_t> sig_to_root; + ISigColumn* sig0 = ctx.get(keys[0])->generate_signature(); + ISigColumn* sig1 = ctx.get(keys[1])->generate_signature(); + for (size_t r_i = 0; r_i < row_num; ++r_i) { + auto cur = std::make_pair(sig0->get_sig(r_i), sig1->get_sig(r_i)); + auto iter = sig_to_root.find(cur); + if (iter == sig_to_root.end()) { + sig_to_root.emplace(cur, offsets.size()); + offsets.push_back(r_i); + std::vector list; + list.push_back(r_i); + to_aggregate.emplace_back(std::move(list)); + } else { + to_aggregate[iter->second].push_back(r_i); + } + } + delete sig0; + delete sig1; + } else { + std::set set; + for (size_t r_i = 0; r_i < row_num; ++r_i) { + std::vector bytes; + Encoder encoder(bytes); + for (size_t k = 0; k < keys.size(); ++k) { + auto val = ctx.get(keys[k])->get_elem(k); + val.encode_sig(val.type(), encoder); + encoder.put_byte('#'); + } + std::string sv(bytes.data(), bytes.size()); + if (set.find(sv) == set.end()) { + offsets.push_back(r_i); + set.insert(sv); + } + } + } + + std::vector> new_columns; + aggregate_value_impl(funcs, to_aggregate, new_columns); + + Context new_ctx; + for (auto col : keys) { + new_ctx.set(col, ctx.get(col)); + } + new_ctx.head = nullptr; + + new_ctx.reshuffle(offsets); + for (size_t k = 0; k < new_columns.size(); ++k) { + auto col = new_columns[k]; + if (col != nullptr) { + new_ctx.set(k, col); + } + } + + new_ctx.head = nullptr; + return new_ctx; + } +}; + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_COMMON_OPERATORS_GROUP_BY_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/operators/intersect.cc b/flex/engines/graph_db/runtime/common/operators/intersect.cc new file mode 100644 index 000000000000..e4e7a9c6e9e7 --- /dev/null +++ b/flex/engines/graph_db/runtime/common/operators/intersect.cc @@ -0,0 +1,181 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/common/operators/intersect.h" +#include "flex/engines/graph_db/runtime/common/columns/edge_columns.h" +#include "flex/engines/graph_db/runtime/common/columns/value_columns.h" +#include "flex/engines/graph_db/runtime/common/columns/vertex_columns.h" + +namespace gs { + +namespace runtime { + +static void ensure_sorted(std::shared_ptr> idx_col, + std::shared_ptr val_col) { + auto& idx_col_ref = *idx_col; + size_t row_num = idx_col_ref.size(); + for (size_t k = 1; k < row_num; ++k) { + CHECK_GE(idx_col_ref.get_value(k), idx_col_ref.get_value(k - 1)); + } +} + +Context Intersect::intersect(Context&& ctx, + std::vector>&& ctxs, + int alias) { + std::vector>, + std::shared_ptr>> + cols; + for (auto& c : ctxs) { + auto& this_ctx = std::get<0>(c); + int idx_col = std::get<1>(c); + int value_col = std::get<2>(c); + cols.emplace_back( + std::dynamic_pointer_cast>(this_ctx.get(idx_col)), + this_ctx.get(value_col)); + } + for (auto& pair : cols) { + ensure_sorted(pair.first, pair.second); + } + if (cols.size() == 2) { + auto& idx_col0 = *cols[0].first; + auto& idx_col1 = *cols[1].first; + + size_t rn0 = idx_col0.size(); + size_t rn1 = idx_col1.size(); + + CHECK(cols[0].second->column_type() == cols[1].second->column_type()); + if (cols[0].second->column_type() == ContextColumnType::kVertex) { + auto vlist0_ptr = + std::dynamic_pointer_cast(cols[0].second); + auto vlist1_ptr = + std::dynamic_pointer_cast(cols[1].second); + if (vlist0_ptr->vertex_column_type() == VertexColumnType::kSingle && + vlist1_ptr->vertex_column_type() == VertexColumnType::kSingle) { + auto& vlist0 = *std::dynamic_pointer_cast(vlist0_ptr); + auto& vlist1 = *std::dynamic_pointer_cast(vlist1_ptr); + + std::vector shuffle_offsets; + SLVertexColumnBuilder builder(*vlist0.get_labels_set().begin()); + + size_t idx0 = 0, idx1 = 0; + std::set lhs_set; + while (idx0 < rn0 && idx1 < rn1) { + if (idx_col0.get_value(idx0) < idx_col1.get_value(idx1)) { + ++idx0; + } else if (idx_col0.get_value(idx0) > idx_col1.get_value(idx1)) { + ++idx1; + } else { + lhs_set.clear(); + size_t common_index = idx_col0.get_value(idx0); + while (idx_col0.get_value(idx0) == common_index) { + lhs_set.insert(vlist0.get_vertex(idx0).second); + ++idx0; + } + while (idx_col1.get_value(idx1) == common_index) { + vid_t cur_v = vlist1.get_vertex(idx1).second; + if (lhs_set.find(cur_v) != lhs_set.end()) { + shuffle_offsets.push_back(common_index); + builder.push_back_opt(cur_v); + } + ++idx1; + } + } + } + + ctx.set_with_reshuffle(alias, builder.finish(), shuffle_offsets); + return ctx; + } + } + } + + LOG(FATAL) << "not support"; +} + +static Context intersect_impl(std::vector&& ctxs, int key) { + if (ctxs[0].get(key)->column_type() == ContextColumnType::kVertex) { + if (ctxs.size() == 2) { + auto& vlist0 = + *(std::dynamic_pointer_cast(ctxs[0].get(key))); + auto& vlist1 = + *(std::dynamic_pointer_cast(ctxs[1].get(key))); + auto& idx_col0 = ctxs[0].get_idx_col(); + auto& idx_col1 = ctxs[1].get_idx_col(); + std::vector offsets0(idx_col0.size()), offsets1(idx_col1.size()); + for (size_t k = 0; k < idx_col0.size(); ++k) { + offsets0[k] = k; + } + for (size_t k = 0; k < idx_col1.size(); ++k) { + offsets1[k] = k; + } + std::sort(offsets0.begin(), offsets0.end(), + [&idx_col0, &vlist0](size_t a, size_t b) { + if (idx_col0.get_value(a) == idx_col0.get_value(b)) { + return vlist0.get_vertex(a) < vlist0.get_vertex(b); + } + return idx_col0.get_value(a) < idx_col0.get_value(b); + }); + std::sort(offsets1.begin(), offsets1.end(), + [&idx_col1, &vlist1](size_t a, size_t b) { + if (idx_col1.get_value(a) == idx_col1.get_value(b)) { + return vlist1.get_vertex(a) < vlist1.get_vertex(b); + } + return idx_col1.get_value(a) < idx_col1.get_value(b); + }); + std::vector shuffle_offsets; + size_t idx0 = 0, idx1 = 0; + while (idx0 < idx_col0.size() && idx1 < idx_col1.size()) { + if (idx_col0.get_value(offsets0[idx0]) < + idx_col1.get_value(offsets1[idx1])) { + ++idx0; + } else if (idx_col0.get_value(offsets0[idx0]) > + idx_col1.get_value(offsets1[idx1])) { + ++idx1; + } else { + auto v0 = vlist0.get_vertex(offsets0[idx0]); + size_t pre_idx1 = idx1; + while (idx1 < idx_col1.size() && + idx_col1.get_value(offsets1[idx1]) == + idx_col0.get_value(offsets0[idx0])) { + auto v1 = vlist1.get_vertex(offsets1[idx1]); + if (v0 == v1) { + shuffle_offsets.push_back(offsets0[idx0]); + } else if (v0 < v1) { + break; + } else { + pre_idx1 = idx1; + } + ++idx1; + } + ++idx0; + idx1 = pre_idx1; + } + } + + ctxs[0].reshuffle(shuffle_offsets); + ctxs[0].pop_idx_col(); + return ctxs[0]; + } + } + LOG(FATAL) << "not support"; + return Context(); +} + +Context Intersect::intersect(std::vector&& ctxs, int key) { + return intersect_impl(std::move(ctxs), key); +} + +} // namespace runtime + +} // namespace gs diff --git a/flex/engines/graph_db/runtime/common/operators/intersect.h b/flex/engines/graph_db/runtime/common/operators/intersect.h new file mode 100644 index 000000000000..597aea5ab2bd --- /dev/null +++ b/flex/engines/graph_db/runtime/common/operators/intersect.h @@ -0,0 +1,41 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RUNTIME_COMMON_OPERATORS_INTERSECT_H_ +#define RUNTIME_COMMON_OPERATORS_INTERSECT_H_ + +#include +#include + +#include "flex/engines/graph_db/runtime/common/context.h" + +namespace gs { + +namespace runtime { + +class Intersect { + public: + static Context intersect(Context&& ctx, + std::vector>&& ctxs, + int alias); + + static Context intersect(std::vector&& ctxs, int key); +}; + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_COMMON_OPERATORS_INTERSECT_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/operators/join.cc b/flex/engines/graph_db/runtime/common/operators/join.cc new file mode 100644 index 000000000000..0441c5b51772 --- /dev/null +++ b/flex/engines/graph_db/runtime/common/operators/join.cc @@ -0,0 +1,185 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/common/operators/join.h" +#include "flex/engines/graph_db/runtime/common/columns/vertex_columns.h" + +namespace gs { + +namespace runtime { +Context Join::join(Context&& ctx, Context&& ctx2, const JoinParams& params) { + CHECK(params.left_columns.size() == params.right_columns.size()) + << "Join columns size mismatch"; + if (params.join_type == JoinKind::kSemiJoin || + params.join_type == JoinKind::kAntiJoin) { + size_t right_size = ctx2.row_num(); + std::set right_set; + std::vector offset; + + for (size_t r_i = 0; r_i < right_size; ++r_i) { + std::vector bytes; + Encoder encoder(bytes); + for (size_t i = 0; i < params.right_columns.size(); i++) { + auto val = ctx2.get(params.right_columns[i])->get_elem(r_i); + val.encode_sig(val.type(), encoder); + encoder.put_byte('#'); + } + std::string cur(bytes.begin(), bytes.end()); + right_set.insert(cur); + } + + size_t left_size = ctx.row_num(); + for (size_t r_i = 0; r_i < left_size; ++r_i) { + std::vector bytes; + Encoder encoder(bytes); + for (size_t i = 0; i < params.left_columns.size(); i++) { + auto val = ctx.get(params.left_columns[i])->get_elem(r_i); + val.encode_sig(val.type(), encoder); + encoder.put_byte('#'); + } + std::string cur(bytes.begin(), bytes.end()); + if (params.join_type == JoinKind::kSemiJoin) { + if (right_set.find(cur) != right_set.end()) { + offset.push_back(r_i); + } + } else { + if (right_set.find(cur) == right_set.end()) { + offset.push_back(r_i); + } + } + } + ctx.reshuffle(offset); + return ctx; + } else if (params.join_type == JoinKind::kInnerJoin) { + size_t right_size = ctx2.row_num(); + std::map> right_set; + std::vector left_offset, right_offset; + + for (size_t r_i = 0; r_i < right_size; ++r_i) { + std::vector bytes; + Encoder encoder(bytes); + for (size_t i = 0; i < params.right_columns.size(); i++) { + auto val = ctx2.get(params.right_columns[i])->get_elem(r_i); + val.encode_sig(val.type(), encoder); + encoder.put_byte('#'); + } + std::string cur(bytes.begin(), bytes.end()); + right_set[cur].emplace_back(r_i); + } + + size_t left_size = ctx.row_num(); + for (size_t r_i = 0; r_i < left_size; ++r_i) { + std::vector bytes; + Encoder encoder(bytes); + for (size_t i = 0; i < params.left_columns.size(); i++) { + auto val = ctx.get(params.left_columns[i])->get_elem(r_i); + val.encode_sig(val.type(), encoder); + encoder.put_byte('#'); + } + std::string cur(bytes.begin(), bytes.end()); + if (right_set.find(cur) != right_set.end()) { + for (auto right : right_set[cur]) { + left_offset.push_back(r_i); + right_offset.push_back(right); + } + } + } + ctx.reshuffle(left_offset); + ctx2.reshuffle(right_offset); + Context ret; + for (size_t i = 0; i < ctx.col_num(); i++) { + ret.set(i, ctx.get(i)); + } + for (size_t i = 0; i < ctx2.col_num(); i++) { + if (i >= ret.col_num() || ret.get(i) == nullptr) { + ret.set(i, ctx2.get(i)); + } + } + return ret; + } else if (params.join_type == JoinKind::kLeftOuterJoin) { + size_t right_size = ctx2.row_num(); + auto right_col = ctx2.get(params.right_columns[0]); + CHECK(right_col->column_type() == ContextColumnType::kVertex); + + std::map> right_map; + for (size_t r_i = 0; r_i < right_size; r_i++) { + std::vector bytes; + Encoder encoder(bytes); + for (size_t i = 0; i < params.right_columns.size(); i++) { + auto val = ctx2.get(params.right_columns[i])->get_elem(r_i); + val.encode_sig(val.type(), encoder); + encoder.put_byte('#'); + } + std::string cur(bytes.begin(), bytes.end()); + right_map[cur].emplace_back(r_i); + } + + std::vector> builders; + for (size_t i = 0; i < ctx2.col_num(); i++) { + if (std::find(params.right_columns.begin(), params.right_columns.end(), + i) == params.right_columns.end() && + ctx2.get(i) != nullptr) { + builders.emplace_back(ctx2.get(i)->optional_builder()); + } else { + builders.emplace_back(nullptr); + } + } + + std::vector offsets; + size_t left_size = ctx.row_num(); + for (size_t r_i = 0; r_i < left_size; r_i++) { + std::vector bytes; + Encoder encoder(bytes); + for (size_t i = 0; i < params.left_columns.size(); i++) { + auto val = ctx.get(params.left_columns[i])->get_elem(r_i); + val.encode_sig(val.type(), encoder); + encoder.put_byte('#'); + } + std::string cur(bytes.begin(), bytes.end()); + if (right_map.find(cur) == right_map.end()) { + for (size_t i = 0; i < ctx2.col_num(); i++) { + if (builders[i] != nullptr) { + builders[i]->push_back_null(); + } + } + offsets.emplace_back(r_i); + } else { + for (auto idx : right_map[cur]) { + for (size_t i = 0; i < ctx2.col_num(); i++) { + if (builders[i] != nullptr) { + builders[i]->push_back_elem(ctx2.get(i)->get_elem(idx)); + } + } + offsets.emplace_back(r_i); + } + } + } + ctx.reshuffle(offsets); + for (size_t i = 0; i < ctx2.col_num(); i++) { + if (builders[i] != nullptr) { + ctx.set(i, builders[i]->finish()); + } else if (i >= ctx.col_num()) { + ctx.set(i, nullptr); + } + } + + return ctx; + } + LOG(FATAL) << "Unsupported join type"; + + return Context(); +} +} // namespace runtime +} // namespace gs \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/operators/join.h b/flex/engines/graph_db/runtime/common/operators/join.h new file mode 100644 index 000000000000..1e8360b3ce58 --- /dev/null +++ b/flex/engines/graph_db/runtime/common/operators/join.h @@ -0,0 +1,38 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RUNTIME_COMMON_OPERATORS_JOIN_H_ +#define RUNTIME_COMMON_OPERATORS_JOIN_H_ + +#include +#include "flex/engines/graph_db/runtime/common/context.h" + +namespace gs { +namespace runtime { + +struct JoinParams { + std::vector left_columns; + std::vector right_columns; + JoinKind join_type; +}; + +class Join { + public: + static Context join(Context&& ctx, Context&& ctx2, const JoinParams& params); +}; +} // namespace runtime +} // namespace gs + +#endif // COMMON_OPERATORS_JOIN_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/operators/order_by.h b/flex/engines/graph_db/runtime/common/operators/order_by.h new file mode 100644 index 000000000000..22c5eaae44cb --- /dev/null +++ b/flex/engines/graph_db/runtime/common/operators/order_by.h @@ -0,0 +1,231 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#ifndef RUNTIME_COMMON_OPERATORS_ORDER_BY_H_ +#define RUNTIME_COMMON_OPERATORS_ORDER_BY_H_ + +#include "flex/engines/graph_db/database/read_transaction.h" +#include "flex/engines/graph_db/runtime/common/context.h" +#include "flex/engines/graph_db/runtime/common/operators/project.h" + +#include + +namespace gs { + +namespace runtime { + +template +class AscValue { + public: + AscValue() : val_() {} + AscValue(const T& val) : val_(val) {} + + bool operator<(const AscValue& rhs) const { return val_ < rhs.val_; } + + const T& value() const { return val_; } + + private: + T val_; +}; + +template +class AscWrapper { + public: + using elem_t = typename VAR_T::elem_t; + using value_t = AscValue; + + AscWrapper(VAR_T&& var) : var_(std::move(var)) {} + AscValue get(size_t idx) const { + return AscValue(var_.typed_eval_path(idx)); + } + + private: + VAR_T var_; +}; + +template +class DescValue { + public: + DescValue() : val_() {} + DescValue(const T& val) : val_(val) {} + + bool operator<(const DescValue& rhs) const { return rhs.val_ < val_; } + + const T& value() const { return val_; } + + private: + T val_; +}; + +template +class DescWrapper { + public: + using elem_t = typename VAR_T::elem_t; + using value_t = DescValue; + + DescWrapper(VAR_T&& var) : var_(std::move(var)) {} + DescValue get(size_t idx) const { + return DescValue(var_.typed_eval_path(idx)); + } + + private: + VAR_T var_; +}; + +bool apply_compare(const std::tuple<>&, size_t lhs, size_t rhs) { + return lhs < rhs; +} + +template +bool apply_compare(const std::tuple& keys, size_t lhs, size_t rhs) { + const T& key = std::get<0>(keys); + + auto lhs_value = key.get(lhs); + auto rhs_value = key.get(rhs); + + if (lhs_value < rhs_value) { + return true; + } else if (rhs_value < lhs_value) { + return false; + } else { + return apply_compare(tail(keys), lhs, rhs); + } +} + +template +struct GeneralTemplatedComparer { + public: + GeneralTemplatedComparer(std::tuple&& keys) + : keys_(std::move(keys)) {} + + bool operator()(size_t lhs, size_t rhs) const { + return apply_compare(keys_, lhs, rhs); + } + + private: + std::tuple keys_; +}; + +template +struct ValueTypeExtractor; + +template +struct ValueTypeExtractor> { + using type = std::tuple; +}; + +template +struct TupleInvokeHelper { + template + static void apply(const std::tuple& input_tuple, + std::tuple& output_tuple, size_t idx) { + std::get(output_tuple) = std::get(input_tuple).typed_eval_path(idx); + TupleInvokeHelper::apply(input_tuple, output_tuple, idx); + } +}; + +// Specialization to end recursion +template +struct TupleInvokeHelper { + template + static void apply(const std::tuple&, std::tuple&, size_t) { + // Do nothing, end of recursion + } +}; + +// Function to start the tuple invocation process +template +typename ValueTypeExtractor>::type invokeTuple( + const std::tuple& input_tuple, size_t idx) { + typename ValueTypeExtractor>::type output_tuple; + TupleInvokeHelper<0, sizeof...(Ts)>::apply(input_tuple, output_tuple, idx); + return output_tuple; +} + +class OrderBy { + public: + template + static void order_by_with_limit(const ReadTransaction& txn, Context& ctx, + const Comparer& cmp, size_t low, + size_t high) { + if (low == 0 && high >= ctx.row_num()) { + std::vector offsets; + for (size_t i = 0; i < ctx.row_num(); ++i) { + offsets.push_back(i); + } + std::sort(offsets.begin(), offsets.end(), + [&](size_t lhs, size_t rhs) { return cmp(lhs, rhs); }); + ctx.reshuffle(offsets); + return; + } + size_t row_num = ctx.row_num(); + std::priority_queue, Comparer> queue(cmp); + for (size_t i = 0; i < row_num; ++i) { + queue.push(i); + if (queue.size() > high) { + queue.pop(); + } + } + std::vector offsets; + for (size_t k = 0; k < low; ++k) { + queue.pop(); + } + offsets.resize(queue.size()); + size_t idx = queue.size(); + + while (!queue.empty()) { + offsets[--idx] = queue.top(); + queue.pop(); + } + + ctx.reshuffle(offsets); + } + + template + static void order_by_with_limit_beta(const ReadTransaction& txn, Context& ctx, + const std::tuple& keys, + size_t low, size_t high) { + size_t row_num = ctx.row_num(); + using value_t = typename ValueTypeExtractor>::type; + std::priority_queue, + std::vector>> + queue; + + for (size_t i = 0; i < row_num; ++i) { + auto cur = invokeTuple(keys, i); + queue.emplace(std::move(cur), i); + if (queue.size() > high) { + queue.pop(); + } + } + + for (size_t k = 0; k < low; ++k) { + queue.pop(); + } + std::vector offsets(queue.size()); + size_t idx = queue.size(); + while (!queue.empty()) { + offsets[--idx] = queue.top().second; + queue.pop(); + } + + ctx.reshuffle(offsets); + } +}; + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_COMMON_OPERATORS_ORDER_BY_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/operators/path_expand.cc b/flex/engines/graph_db/runtime/common/operators/path_expand.cc new file mode 100644 index 000000000000..4117a8e4d8bf --- /dev/null +++ b/flex/engines/graph_db/runtime/common/operators/path_expand.cc @@ -0,0 +1,393 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/common/operators/path_expand.h" + +namespace gs { + +namespace runtime { + +Context PathExpand::edge_expand_v(const ReadTransaction& txn, Context&& ctx, + const PathExpandParams& params) { + std::vector shuffle_offset; + if (params.labels.size() == 1) { + if (params.dir == Direction::kOut) { + auto& input_vertex_list = + *std::dynamic_pointer_cast(ctx.get(params.start_tag)); + label_t output_vertex_label = params.labels[0].dst_label; + label_t edge_label = params.labels[0].edge_label; + SLVertexColumnBuilder builder(output_vertex_label); + + std::vector input; + std::vector output; + input_vertex_list.foreach_vertex( + [&](size_t index, label_t label, vid_t v) { + int depth = 0; + input.clear(); + output.clear(); + input.push_back(v); + while (depth < params.hop_upper && !input.empty()) { + if (depth >= params.hop_lower) { + for (auto u : input) { + builder.push_back_opt(u); + shuffle_offset.push_back(index); + + auto oe_iter = txn.GetOutEdgeIterator( + label, u, output_vertex_label, edge_label); + while (oe_iter.IsValid()) { + output.push_back(oe_iter.GetNeighbor()); + oe_iter.Next(); + } + } + } else { + for (auto u : input) { + auto oe_iter = txn.GetOutEdgeIterator( + label, u, output_vertex_label, edge_label); + while (oe_iter.IsValid()) { + output.push_back(oe_iter.GetNeighbor()); + oe_iter.Next(); + } + } + } + ++depth; + input.clear(); + std::swap(input, output); + } + }); + + ctx.set_with_reshuffle_beta(params.alias, builder.finish(), + shuffle_offset, params.keep_cols); + return ctx; + } else if (params.dir == Direction::kBoth && + params.labels[0].src_label == params.labels[0].dst_label) { + auto& input_vertex_list = + *std::dynamic_pointer_cast(ctx.get(params.start_tag)); + label_t output_vertex_label = params.labels[0].dst_label; + label_t edge_label = params.labels[0].edge_label; + + SLVertexColumnBuilder builder(output_vertex_label); + + std::vector> input; + std::vector> output; + std::set exclude; + CHECK_GE(params.hop_lower, 0); + CHECK_GE(params.hop_upper, params.hop_lower); + if (params.hop_lower == 0) { + LOG(FATAL) << "xxx"; + } else { + if (params.hop_upper == 1) { + LOG(FATAL) << "xxx"; + } else { + input_vertex_list.foreach_vertex( + [&](size_t index, label_t label, vid_t v) { + output.emplace_back(index, v); + }); + } + int depth = 0; + while (depth < params.hop_upper) { + input.clear(); + std::swap(input, output); + + if (depth >= params.hop_lower) { + for (auto& pair : input) { + builder.push_back_opt(pair.second); + shuffle_offset.push_back(pair.first); + } + } + + if (depth + 1 >= params.hop_upper) { + break; + } + + auto label = params.labels[0].src_label; + for (auto& pair : input) { + auto index = pair.first; + auto v = pair.second; + auto oe_iter = txn.GetOutEdgeIterator(label, v, output_vertex_label, + edge_label); + while (oe_iter.IsValid()) { + auto nbr = oe_iter.GetNeighbor(); + if (exclude.find(nbr) == exclude.end()) { + output.emplace_back(index, nbr); + } + oe_iter.Next(); + } + + auto ie_iter = txn.GetInEdgeIterator(label, v, output_vertex_label, + edge_label); + while (ie_iter.IsValid()) { + auto nbr = ie_iter.GetNeighbor(); + if (exclude.find(nbr) == exclude.end()) { + output.emplace_back(index, nbr); + } + ie_iter.Next(); + } + } + + ++depth; + } + } + ctx.set_with_reshuffle_beta(params.alias, builder.finish(), + shuffle_offset, params.keep_cols); + return ctx; + } + } else { + if (params.dir == Direction::kOut) { + auto& input_vertex_list = + *std::dynamic_pointer_cast(ctx.get(params.start_tag)); + std::set labels; + for (auto& label : params.labels) { + labels.emplace(label.dst_label); + } + + MLVertexColumnBuilder builder(labels); + std::vector> input; + std::vector> output; + input_vertex_list.foreach_vertex( + [&](size_t index, label_t label, vid_t v) { + output.emplace_back(label, v, index); + }); + int depth = 0; + while (depth < params.hop_upper) { + input.clear(); + std::swap(input, output); + if (depth >= params.hop_lower) { + for (auto& tuple : input) { + builder.push_back_vertex( + std::make_pair(std::get<0>(tuple), std::get<1>(tuple))); + shuffle_offset.push_back(std::get<2>(tuple)); + } + } + + if (depth + 1 >= params.hop_upper) { + break; + } + + for (auto& tuple : input) { + auto label = std::get<0>(tuple); + auto v = std::get<1>(tuple); + auto index = std::get<2>(tuple); + for (auto& label_triplet : params.labels) { + if (label_triplet.src_label == label) { + auto oe_iter = txn.GetOutEdgeIterator(label_triplet.src_label, v, + label_triplet.dst_label, + label_triplet.edge_label); + + while (oe_iter.IsValid()) { + auto nbr = oe_iter.GetNeighbor(); + output.emplace_back(label_triplet.dst_label, nbr, index); + oe_iter.Next(); + } + } + } + } + ++depth; + } + ctx.set_with_reshuffle_beta(params.alias, builder.finish(), + shuffle_offset, params.keep_cols); + return ctx; + } else if (params.dir == Direction::kBoth) { + auto& input_vertex_list = + *std::dynamic_pointer_cast(ctx.get(params.start_tag)); + std::set labels; + for (auto& label : params.labels) { + labels.emplace(label.dst_label); + } + + MLVertexColumnBuilder builder(labels); + std::vector> input; + std::vector> output; + input_vertex_list.foreach_vertex( + [&](size_t index, label_t label, vid_t v) { + output.emplace_back(label, v, index); + }); + int depth = 0; + while (depth < params.hop_upper) { + input.clear(); + std::swap(input, output); + if (depth >= params.hop_lower) { + for (auto& tuple : input) { + builder.push_back_vertex( + std::make_pair(std::get<0>(tuple), std::get<1>(tuple))); + shuffle_offset.push_back(std::get<2>(tuple)); + } + } + + if (depth + 1 >= params.hop_upper) { + break; + } + + for (auto& tuple : input) { + auto label = std::get<0>(tuple); + auto v = std::get<1>(tuple); + auto index = std::get<2>(tuple); + for (auto& label_triplet : params.labels) { + if (label_triplet.src_label == label) { + auto oe_iter = txn.GetOutEdgeIterator(label_triplet.src_label, v, + label_triplet.dst_label, + label_triplet.edge_label); + + while (oe_iter.IsValid()) { + auto nbr = oe_iter.GetNeighbor(); + output.emplace_back(label_triplet.dst_label, nbr, index); + oe_iter.Next(); + } + } + if (label_triplet.dst_label == label) { + auto ie_iter = txn.GetInEdgeIterator(label_triplet.dst_label, v, + label_triplet.src_label, + label_triplet.edge_label); + while (ie_iter.IsValid()) { + auto nbr = ie_iter.GetNeighbor(); + output.emplace_back(label_triplet.src_label, nbr, index); + ie_iter.Next(); + } + } + } + } + depth++; + } + ctx.set_with_reshuffle_beta(params.alias, builder.finish(), + shuffle_offset, params.keep_cols); + return ctx; + } + } + LOG(FATAL) << "not support..."; + return ctx; +} + +Context PathExpand::edge_expand_p(const ReadTransaction& txn, Context&& ctx, + const PathExpandParams& params) { + std::vector shuffle_offset; + auto& input_vertex_list = + *std::dynamic_pointer_cast(ctx.get(params.start_tag)); + auto label_sets = input_vertex_list.get_labels_set(); + auto labels = params.labels; + auto dir = params.dir; + std::vector, size_t>> input; + std::vector, size_t>> output; + std::vector> path_impls; + + GeneralPathColumnBuilder builder; + if (dir == Direction::kOut) { + foreach_vertex(input_vertex_list, + [&](size_t index, label_t label, vid_t v) { + auto p = PathImpl::make_path_impl(label, v); + input.emplace_back(p, index); + }); + int depth = 0; + while (depth < params.hop_upper) { + output.clear(); + if (depth >= params.hop_lower) { + for (auto& [path, index] : input) { + builder.push_back_opt(Path::make_path(path)); + path_impls.emplace_back(path); + shuffle_offset.push_back(index); + } + } + if (depth + 1 >= params.hop_upper) { + break; + } + + for (auto& [path, index] : input) { + auto end = path->get_end(); + for (auto& label_triplet : labels) { + if (label_triplet.src_label == end.first) { + auto oe_iter = txn.GetOutEdgeIterator(end.first, end.second, + label_triplet.dst_label, + label_triplet.edge_label); + while (oe_iter.IsValid()) { + std::shared_ptr new_path = + path->expand(label_triplet.dst_label, oe_iter.GetNeighbor()); + output.emplace_back(new_path, index); + oe_iter.Next(); + } + } + } + } + + input.clear(); + std::swap(input, output); + ++depth; + } + builder.set_path_impls(path_impls); + ctx.set_with_reshuffle_beta(params.alias, builder.finish(), shuffle_offset, + params.keep_cols); + + return ctx; + } else if (dir == Direction::kBoth) { + foreach_vertex(input_vertex_list, + [&](size_t index, label_t label, vid_t v) { + auto p = PathImpl::make_path_impl(label, v); + input.emplace_back(p, index); + }); + int depth = 0; + while (depth < params.hop_upper) { + output.clear(); + if (depth >= params.hop_lower) { + for (auto& [path, index] : input) { + builder.push_back_opt(Path::make_path(path)); + path_impls.emplace_back(path); + shuffle_offset.push_back(index); + } + } + if (depth + 1 >= params.hop_upper) { + break; + } + + for (auto& [path, index] : input) { + auto end = path->get_end(); + for (auto& label_triplet : labels) { + if (label_triplet.src_label == end.first) { + auto oe_iter = txn.GetOutEdgeIterator(end.first, end.second, + label_triplet.dst_label, + label_triplet.edge_label); + while (oe_iter.IsValid()) { + auto new_path = + path->expand(label_triplet.dst_label, oe_iter.GetNeighbor()); + output.emplace_back(new_path, index); + oe_iter.Next(); + } + } + if (label_triplet.dst_label == end.first) { + auto ie_iter = txn.GetInEdgeIterator(end.first, end.second, + label_triplet.src_label, + label_triplet.edge_label); + while (ie_iter.IsValid()) { + auto new_path = + path->expand(label_triplet.src_label, ie_iter.GetNeighbor()); + output.emplace_back(new_path, index); + ie_iter.Next(); + } + } + } + } + + input.clear(); + std::swap(input, output); + ++depth; + } + builder.set_path_impls(path_impls); + ctx.set_with_reshuffle_beta(params.alias, builder.finish(), shuffle_offset, + params.keep_cols); + return ctx; + } + LOG(FATAL) << "not support..."; + return ctx; +} + +} // namespace runtime + +} // namespace gs diff --git a/flex/engines/graph_db/runtime/common/operators/path_expand.h b/flex/engines/graph_db/runtime/common/operators/path_expand.h new file mode 100644 index 000000000000..8a70326428e4 --- /dev/null +++ b/flex/engines/graph_db/runtime/common/operators/path_expand.h @@ -0,0 +1,160 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RUNTIME_COMMON_OPERATORS_PATH_EXPAND_H_ +#define RUNTIME_COMMON_OPERATORS_PATH_EXPAND_H_ + +#include + +#include "flex/engines/graph_db/database/read_transaction.h" +#include "flex/engines/graph_db/runtime/common/columns/path_columns.h" +#include "flex/engines/graph_db/runtime/common/columns/vertex_columns.h" +#include "flex/engines/graph_db/runtime/common/context.h" +#include "flex/engines/graph_db/runtime/common/types.h" + +namespace gs { + +namespace runtime { + +struct PathExpandParams { + int start_tag; + std::vector labels; + int alias; + Direction dir; + int hop_lower; + int hop_upper; + std::set keep_cols; +}; + +class PathExpand { + public: + // PathExpand(expandOpt == Vertex && alias == -1 && resultOpt == END_V) + + // GetV(opt == END) + static Context edge_expand_v(const ReadTransaction& txn, Context&& ctx, + const PathExpandParams& params); + static Context edge_expand_p(const ReadTransaction& txn, Context&& ctx, + const PathExpandParams& params); + + template + static Context edge_expand_v_pred(const ReadTransaction& txn, Context&& ctx, + const PathExpandParams& params, + const PRED_T& pred) { + std::vector shuffle_offset; + if (params.labels.size() == 1 && + params.labels[0].src_label == params.labels[0].dst_label) { + if (params.dir == Direction::kOut) { + auto& input_vertex_list = *std::dynamic_pointer_cast( + ctx.get(params.start_tag)); + label_t output_vertex_label = params.labels[0].dst_label; + label_t edge_label = params.labels[0].edge_label; + label_t vertex_label = params.labels[0].src_label; + SLVertexColumnBuilder builder(output_vertex_label); + +#if 0 + std::vector input; + std::vector output; + input_vertex_list.foreach_vertex( + [&](size_t index, label_t label, vid_t v) { + int depth = 0; + input.clear(); + output.clear(); + input.push_back(v); + while (depth < params.hop_upper && !input.empty()) { + if (depth >= params.hop_lower) { + for (auto u : input) { + if (pred(label, u)) { + builder.push_back_opt(u); + shuffle_offset.push_back(index); + } + + auto oe_iter = txn.GetOutEdgeIterator( + label, u, output_vertex_label, edge_label); + while (oe_iter.IsValid()) { + output.push_back(oe_iter.GetNeighbor()); + oe_iter.Next(); + } + } + } else { + for (auto u : input) { + auto oe_iter = txn.GetOutEdgeIterator( + label, u, output_vertex_label, edge_label); + while (oe_iter.IsValid()) { + output.push_back(oe_iter.GetNeighbor()); + oe_iter.Next(); + } + } + } + ++depth; + input.clear(); + std::swap(input, output); + } + }); +#else + std::vector> input; + std::vector> output; + input_vertex_list.foreach_vertex( + [&](size_t index, label_t label, vid_t v) { + output.emplace_back(index, v); + }); + int depth = 0; + auto oe_csr = txn.GetOutgoingSingleImmutableGraphView( + vertex_label, vertex_label, edge_label); + while (depth < params.hop_upper && !output.empty()) { + input.clear(); + std::swap(input, output); + + for (auto& pair : input) { + if (pred(vertex_label, pair.second, pair.first)) { + builder.push_back_opt(pair.second); + shuffle_offset.push_back(pair.first); + } + } + if (depth + 1 >= params.hop_upper) { + break; + } + for (auto& pair : input) { + auto index = pair.first; + auto v = pair.second; + if (oe_csr.exist(v)) { + output.emplace_back(index, oe_csr.get_edge(v).neighbor); + } + // auto oe_iter = txn.GetOutEdgeIterator( + // vertex_label, v, output_vertex_label, edge_label); + // while (oe_iter.IsValid()) { + // auto nbr = oe_iter.GetNeighbor(); + // output.emplace_back(index, nbr); + // oe_iter.Next(); + // } + } + + ++depth; + } +#endif + + ctx.set_with_reshuffle_beta(params.alias, builder.finish(), + shuffle_offset, params.keep_cols); + return ctx; + } + } + LOG(FATAL) << "not support..."; + return ctx; + } +}; + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_COMMON_OPERATORS_PATH_EXPAND_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/operators/project.h b/flex/engines/graph_db/runtime/common/operators/project.h new file mode 100644 index 000000000000..907f666bd77e --- /dev/null +++ b/flex/engines/graph_db/runtime/common/operators/project.h @@ -0,0 +1,159 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RUNTIME_COMMON_OPERATORS_PROJECT_H_ +#define RUNTIME_COMMON_OPERATORS_PROJECT_H_ + +#include +#include + +#include "flex/engines/graph_db/runtime/common/columns/i_context_column.h" +#include "flex/engines/graph_db/runtime/common/columns/value_columns.h" + +namespace gs { + +namespace runtime { + +template +struct ProjectExpr { + ProjectExpr(const Expr& expr_, int alias_) : expr(expr_), alias(alias_) {} + + const Expr& expr; + int alias; +}; + +struct DummyGetter { + DummyGetter(int from_, int to_) : from(from_), to(to_) {} + + int from; + int to; +}; + +template +auto tail_impl(std::index_sequence, const std::tuple& t) { + return std::make_tuple(std::get(t)...); +} + +template +auto tail(const std::tuple& t) { + return tail_impl(std::make_index_sequence(), t); +} + +void map_value_impl(Context& ctx, const std::tuple<>&, size_t row_num, + std::vector>& output) {} + +template +void map_value_impl(Context& ctx, const std::tuple, Rest...>& t, + size_t row_num, + std::vector>& output) { + const ProjectExpr& cur = std::get<0>(t); + using ELEM_T = typename T::elem_t; + + ValueColumnBuilder builder; + builder.reserve(row_num); + for (size_t k = 0; k < row_num; ++k) { + builder.push_back_elem(cur.expr.eval_path(k)); + } + + if (output.size() <= cur.alias) { + output.resize(cur.alias + 1, nullptr); + } + output[cur.alias] = builder.finish(); + + map_value_impl(ctx, tail(t), row_num, output); +} + +template +void map_value_impl(Context& ctx, const std::tuple& t, + size_t row_num, + std::vector>& output) { + const DummyGetter& getter = std::get<0>(t); + if (output.size() <= getter.to) { + output.resize(getter.to + 1, nullptr); + } + output[getter.to] = ctx.get(getter.from); + + map_value_impl(ctx, tail(t), row_num, output); +} + +class Project { + public: + static Context select_column( + Context&& ctx, const std::vector>& mappings) { + Context new_ctx; + for (auto& pair : mappings) { + new_ctx.set(pair.second, ctx.get(pair.first)); + } + new_ctx.head = ctx.head; + return new_ctx; + } + + template + static Context map_value(const ReadTransaction& txn, Context&& ctx, + const std::tuple& exprs, bool is_append) { + std::vector> new_columns; + map_value_impl(ctx, exprs, ctx.row_num(), new_columns); + + Context new_ctx; + if (is_append) { + size_t col_num = ctx.col_num(); + for (size_t k = 0; k < col_num; ++k) { + auto col = ctx.get(k); + if (col != nullptr) { + new_ctx.set(k, col); + } + } + } + size_t new_col_num = new_columns.size(); + for (size_t k = 0; k < new_col_num; ++k) { + auto col = new_columns[k]; + if (col != nullptr) { + new_ctx.set(k, col); + } + } + + if (is_append) { + new_ctx.head = ctx.head; + } else { + new_ctx.head = nullptr; + } + return new_ctx; + } + + template + static Context map_value_general( + const ReadTransaction& txn, Context&& ctx, + const std::vector>& expressions, bool is_append) { + if (!is_append) { + std::vector> new_columns; + for (auto& pr : expressions) { + int alias = pr.alias; + const auto& expr = pr.expr; + if (new_columns.size() <= alias) { + new_columns.resize(alias + 1, nullptr); + } + } + } + + LOG(FATAL) << "not support"; + return ctx; + } +}; + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_COMMON_OPERATORS_PROJECT_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/operators/scan.cc b/flex/engines/graph_db/runtime/common/operators/scan.cc new file mode 100644 index 000000000000..ffe58d4dbb16 --- /dev/null +++ b/flex/engines/graph_db/runtime/common/operators/scan.cc @@ -0,0 +1,58 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/common/operators/scan.h" + +namespace gs { +namespace runtime { + +Context Scan::find_vertex(const ReadTransaction& txn, label_t label, + const Any& pk, int alias, bool scan_oid) { + if (scan_oid) { + SLVertexColumnBuilder builder(label); + vid_t vid; + if (txn.GetVertexIndex(label, pk, vid)) { + builder.push_back_opt(vid); + } + Context ctx; + ctx.set(alias, builder.finish()); + return ctx; + } else { + SLVertexColumnBuilder builder(label); + vid_t vid{}; + int64_t gid{}; + if (pk.type == PropertyType::kInt64) { + gid = pk.AsInt64(); + } else if (pk.type == PropertyType::kInt32) { + gid = pk.AsInt32(); + } else { + LOG(FATAL) << "Unsupported primary key type"; + } + if (GlobalId::get_label_id(gid) == label) { + vid = GlobalId::get_vid(gid); + } else { + LOG(ERROR) << "Global id " << gid << " does not match label " << label; + return Context(); + } + builder.push_back_opt(vid); + Context ctx; + ctx.set(alias, builder.finish()); + return ctx; + } +} + +} // namespace runtime + +} // namespace gs diff --git a/flex/engines/graph_db/runtime/common/operators/scan.h b/flex/engines/graph_db/runtime/common/operators/scan.h new file mode 100644 index 000000000000..13265a0be01d --- /dev/null +++ b/flex/engines/graph_db/runtime/common/operators/scan.h @@ -0,0 +1,106 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RUNTIME_COMMON_OPERATORS_SCAN_H_ +#define RUNTIME_COMMON_OPERATORS_SCAN_H_ + +#include + +#include "flex/engines/graph_db/runtime/common/columns/vertex_columns.h" +#include "flex/engines/graph_db/runtime/common/context.h" + +namespace gs { + +namespace runtime { + +struct ScanParams { + int alias; + std::vector tables; +}; + +class Scan { + public: + template + static Context scan_vertex(const ReadTransaction& txn, + const ScanParams& params, + const PRED_T& predicate) { + Context ctx; + if (params.tables.size() == 1) { + label_t label = params.tables[0]; + SLVertexColumnBuilder builder(label); + vid_t vnum = txn.GetVertexNum(label); + for (vid_t vid = 0; vid != vnum; ++vid) { + if (predicate(label, vid)) { + builder.push_back_opt(vid); + } + } + ctx.set(params.alias, builder.finish()); + } else if (params.tables.size() > 1) { + MLVertexColumnBuilder builder; + + for (auto label : params.tables) { + vid_t vnum = txn.GetVertexNum(label); + for (vid_t vid = 0; vid != vnum; ++vid) { + if (predicate(label, vid)) { + builder.push_back_vertex(std::make_pair(label, vid)); + } + } + } + ctx.set(params.alias, builder.finish()); + } + return ctx; + } + + template + static Context scan_gid_vertex(const ReadTransaction& txn, + const ScanParams& params, + const PRED_T& predicate, + const std::vector& gids) { + Context ctx; + if (params.tables.size() == 1) { + label_t label = params.tables[0]; + SLVertexColumnBuilder builder(label); + for (auto gid : gids) { + vid_t vid = GlobalId::get_vid(gid); + if (GlobalId::get_label_id(gid) == label && predicate(label, vid)) { + builder.push_back_opt(vid); + } + } + ctx.set(params.alias, builder.finish()); + } else if (params.tables.size() > 1) { + MLVertexColumnBuilder builder; + + for (auto label : params.tables) { + for (auto gid : gids) { + vid_t vid = GlobalId::get_vid(gid); + if (GlobalId::get_label_id(gid) == label && predicate(label, vid)) { + builder.push_back_vertex(std::make_pair(label, vid)); + } + } + } + ctx.set(params.alias, builder.finish()); + } + return ctx; + } + + static Context find_vertex(const ReadTransaction& txn, label_t label, + const Any& pk, int alias, bool scan_oid); +}; + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_COMMON_OPERATORS_SCAN_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/operators/select.h b/flex/engines/graph_db/runtime/common/operators/select.h new file mode 100644 index 000000000000..47a98a7b1eac --- /dev/null +++ b/flex/engines/graph_db/runtime/common/operators/select.h @@ -0,0 +1,45 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RUNTIME_COMMON_OPERATORS_SELECT_H_ +#define RUNTIME_COMMON_OPERATORS_SELECT_H_ + +#include "flex/engines/graph_db/runtime/common/context.h" + +namespace gs { + +namespace runtime { + +class Select { + public: + template + static void select(Context& ctx, const PRED_T& pred) { + size_t row_num = ctx.row_num(); + std::vector offsets; + for (size_t k = 0; k < row_num; ++k) { + if (pred(k)) { + offsets.push_back(k); + } + } + + ctx.reshuffle(offsets); + } +}; + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_COMMON_OPERATORS_SELECT_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/operators/sink.cc b/flex/engines/graph_db/runtime/common/operators/sink.cc new file mode 100644 index 000000000000..550e827a88d6 --- /dev/null +++ b/flex/engines/graph_db/runtime/common/operators/sink.cc @@ -0,0 +1,39 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/common/operators/sink.h" + +namespace gs { + +namespace runtime { +// TODO: Implement the sink function +void Sink::sink(const Context& ctx, Encoder& output) { + size_t row_num = ctx.row_num(); + size_t col_num = ctx.col_num(); + for (size_t i = 0; i < row_num; ++i) { + for (size_t j = 0; j < col_num; ++j) { + auto col = ctx.get(j); + if (col == nullptr) { + continue; + } + auto val = col->get_elem(row_num - i - 1); + // val.sink(output); + } + } +} + +} // namespace runtime + +} // namespace gs diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/InvalidSchemaException.java b/flex/engines/graph_db/runtime/common/operators/sink.h similarity index 51% rename from interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/InvalidSchemaException.java rename to flex/engines/graph_db/runtime/common/operators/sink.h index 3faffc91107c..2e282c3876ea 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/InvalidSchemaException.java +++ b/flex/engines/graph_db/runtime/common/operators/sink.h @@ -1,11 +1,10 @@ -/** - * Copyright 2020 Alibaba Group Holding Limited. +/** Copyright 2020 Alibaba Group Holding Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,11 +12,22 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.alibaba.graphscope.groot.common.exception; -public class InvalidSchemaException extends GrootException { +#ifndef RUNTIME_COMMON_OPERATORS_SINK_H_ +#define RUNTIME_COMMON_OPERATORS_SINK_H_ - public InvalidSchemaException(String msg) { - super(msg); - } -} +#include "flex/engines/graph_db/runtime/common/context.h" +#include "flex/utils/app_utils.h" + +namespace gs { +namespace runtime { + +class Sink { + public: + static void sink(const Context& ctx, Encoder& output); +}; + +} // namespace runtime +} // namespace gs + +#endif // RUNTIME_COMMON_OPERATORS_SINK_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/operators/union.cc b/flex/engines/graph_db/runtime/common/operators/union.cc new file mode 100644 index 000000000000..72fefaed9f66 --- /dev/null +++ b/flex/engines/graph_db/runtime/common/operators/union.cc @@ -0,0 +1,38 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/common/operators/union.h" + +namespace gs { + +namespace runtime { + +Context Union::union_op(Context&& ctx1, Context&& ctx2) { + CHECK(ctx1.col_num() == ctx2.col_num()); + size_t col_num = ctx1.col_num(); + Context ret; + for (size_t col_i = 0; col_i < col_num; ++col_i) { + if (ctx1.get(col_i) == nullptr) { + CHECK(ctx2.get(col_i) == nullptr); + continue; + } + ret.set(col_i, ctx1.get(col_i)->union_col(ctx2.get(col_i))); + } + return ret; +} + +} // namespace runtime + +} // namespace gs diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/JSONException.java b/flex/engines/graph_db/runtime/common/operators/union.h similarity index 53% rename from interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/JSONException.java rename to flex/engines/graph_db/runtime/common/operators/union.h index 836ba9bd6d8a..27d3789ea4b0 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/JSONException.java +++ b/flex/engines/graph_db/runtime/common/operators/union.h @@ -1,11 +1,10 @@ -/** - * Copyright 2020 Alibaba Group Holding Limited. +/** Copyright 2020 Alibaba Group Holding Limited. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -13,19 +12,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.alibaba.graphscope.groot.common.exception; -public class JSONException extends RuntimeException { +#ifndef RUNTIME_COMMON_OPERATORS_UNION_H_ +#define RUNTIME_COMMON_OPERATORS_UNION_H_ - public JSONException(String msg) { - super(msg); - } +#include "flex/engines/graph_db/runtime/common/context.h" - public JSONException(String msg, Throwable e) { - super(msg, e); - } +namespace gs { - public JSONException(Throwable e) { - super(e); - } -} +namespace runtime { + +class Union { + public: + static Context union_op(Context&& ctx1, Context&& ctx2); +}; + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_COMMON_OPERATORS_UNION_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/rt_any.cc b/flex/engines/graph_db/runtime/common/rt_any.cc new file mode 100644 index 000000000000..a47b6cced824 --- /dev/null +++ b/flex/engines/graph_db/runtime/common/rt_any.cc @@ -0,0 +1,777 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/common/rt_any.h" + +namespace gs { + +namespace runtime { + +const RTAnyType RTAnyType::kVertex = + RTAnyType(RTAnyType::RTAnyTypeImpl::kVertex); +const RTAnyType RTAnyType::kEdge = RTAnyType(RTAnyType::RTAnyTypeImpl::kEdge); +const RTAnyType RTAnyType::kI64Value = + RTAnyType(RTAnyType::RTAnyTypeImpl::kI64Value); +const RTAnyType RTAnyType::kU64Value = + RTAnyType(RTAnyType::RTAnyTypeImpl::kU64Value); +const RTAnyType RTAnyType::kI32Value = + RTAnyType(RTAnyType::RTAnyTypeImpl::kI32Value); +const RTAnyType RTAnyType::kF64Value = + RTAnyType(RTAnyType::RTAnyTypeImpl::kF64Value); + +const RTAnyType RTAnyType::kBoolValue = + RTAnyType(RTAnyType::RTAnyTypeImpl::kBoolValue); +const RTAnyType RTAnyType::kStringValue = + RTAnyType(RTAnyType::RTAnyTypeImpl::kStringValue); +const RTAnyType RTAnyType::kVertexSetValue = + RTAnyType(RTAnyType::RTAnyTypeImpl::kVertexSetValue); +const RTAnyType RTAnyType::kStringSetValue = + RTAnyType(RTAnyType::RTAnyTypeImpl::kStringSetValue); +const RTAnyType RTAnyType::kUnknown = + RTAnyType(RTAnyType::RTAnyTypeImpl::kUnknown); +const RTAnyType RTAnyType::kDate32 = + RTAnyType(RTAnyType::RTAnyTypeImpl::kDate32); +const RTAnyType RTAnyType::kPath = RTAnyType(RTAnyType::RTAnyTypeImpl::kPath); +const RTAnyType RTAnyType::kNull = RTAnyType(RTAnyType::RTAnyTypeImpl::kNull); +const RTAnyType RTAnyType::kTuple = RTAnyType(RTAnyType::RTAnyTypeImpl::kTuple); +const RTAnyType RTAnyType::kList = RTAnyType(RTAnyType::RTAnyTypeImpl::kList); +const RTAnyType RTAnyType::kMap = RTAnyType(RTAnyType::RTAnyTypeImpl::kMap); +RTAny List::get(size_t idx) const { return impl_->get(idx); } +RTAnyType parse_from_ir_data_type(const ::common::IrDataType& dt) { + switch (dt.type_case()) { + case ::common::IrDataType::TypeCase::kDataType: { + const ::common::DataType ddt = dt.data_type(); + switch (ddt) { + case ::common::DataType::BOOLEAN: + return RTAnyType::kBoolValue; + case ::common::DataType::INT64: + return RTAnyType::kI64Value; + case ::common::DataType::STRING: + return RTAnyType::kStringValue; + case ::common::DataType::INT32: + return RTAnyType::kI32Value; + case ::common::DataType::DATE32: + return RTAnyType::kDate32; + case ::common::DataType::STRING_ARRAY: + return RTAnyType::kStringSetValue; + case ::common::DataType::TIMESTAMP: + return RTAnyType::kDate32; + case ::common::DataType::DOUBLE: + return RTAnyType::kF64Value; + default: + LOG(FATAL) << "unrecoginized data type - " << ddt; + break; + } + } break; + case ::common::IrDataType::TypeCase::kGraphType: { + const ::common::GraphDataType gdt = dt.graph_type(); + switch (gdt.element_opt()) { + case ::common::GraphDataType_GraphElementOpt:: + GraphDataType_GraphElementOpt_VERTEX: + return RTAnyType::kVertex; + case ::common::GraphDataType_GraphElementOpt:: + GraphDataType_GraphElementOpt_EDGE: + return RTAnyType::kEdge; + default: + LOG(FATAL) << "unrecoginized graph data type"; + break; + } + } break; + default: + break; + } + + // LOG(FATAL) << "unknown"; + return RTAnyType::kUnknown; +} + +RTAny::RTAny() : type_(RTAnyType::kUnknown), value_() {} +RTAny::RTAny(RTAnyType type) : type_(type) {} + +RTAny::RTAny(const Any& val) { + if (val.type == PropertyType::Int64()) { + type_ = RTAnyType::kI64Value; + value_.i64_val = val.AsInt64(); + } else if (val.type == PropertyType::String()) { + type_ = RTAnyType::kStringValue; + value_.str_val = val.AsStringView(); + } else if (val.type == PropertyType::Date()) { + type_ = RTAnyType::kI64Value; + value_.i64_val = val.AsDate().milli_second; + } else if (val.type == PropertyType::Int32()) { + type_ = RTAnyType::kI32Value; + value_.i32_val = val.AsInt32(); + } else if (val.type == PropertyType::kDouble) { + type_ = RTAnyType::kF64Value; + value_.f64_val = val.AsDouble(); + } else if (val.type == PropertyType::Bool()) { + type_ = RTAnyType::kBoolValue; + value_.b_val = val.AsBool(); + } else { + LOG(FATAL) << "Any value: " << val.to_string() + << ", type = " << val.type.type_enum; + } +} + +RTAny::RTAny(const Path& p) { + type_ = RTAnyType::kPath; + value_.p = p; +} +RTAny::RTAny(const RTAny& rhs) : type_(rhs.type_) { + if (type_ == RTAnyType::kBoolValue) { + value_.b_val = rhs.value_.b_val; + } else if (type_ == RTAnyType::kI64Value) { + value_.i64_val = rhs.value_.i64_val; + } else if (type_ == RTAnyType::kI32Value) { + value_.i32_val = rhs.value_.i32_val; + } else if (type_ == RTAnyType::kVertex) { + value_.vertex = rhs.value_.vertex; + } else if (type_ == RTAnyType::kStringValue) { + value_.str_val = rhs.value_.str_val; + } else if (type_ == RTAnyType::kNull) { + // do nothing + } else if (type_ == RTAnyType::kTuple) { + value_.t = rhs.value_.t.dup(); + } else if (type_ == RTAnyType::kList) { + value_.list = rhs.value_.list; + } else if (type_ == RTAnyType::kF64Value) { + value_.f64_val = rhs.value_.f64_val; + } else if (type_ == RTAnyType::kMap) { + value_.map = rhs.value_.map; + } else { + LOG(FATAL) << "unexpected type: " << static_cast(type_.type_enum_); + } +} + +RTAny& RTAny::operator=(const RTAny& rhs) { + type_ = rhs.type_; + if (type_ == RTAnyType::kBoolValue) { + value_.b_val = rhs.value_.b_val; + } else if (type_ == RTAnyType::kI64Value) { + value_.i64_val = rhs.value_.i64_val; + } else if (type_ == RTAnyType::kI32Value) { + value_.i32_val = rhs.value_.i32_val; + } else if (type_ == RTAnyType::kVertex) { + value_.vertex = rhs.value_.vertex; + } else if (type_ == RTAnyType::kStringValue) { + value_.str_val = rhs.value_.str_val; + } else if (type_ == RTAnyType::kTuple) { + value_.t = rhs.value_.t.dup(); + } else if (type_ == RTAnyType::kList) { + value_.list = rhs.value_.list; + } else if (type_ == RTAnyType::kF64Value) { + value_.f64_val = rhs.value_.f64_val; + } else if (type_ == RTAnyType::kMap) { + value_.map = rhs.value_.map; + } else { + LOG(FATAL) << "unexpected type: " << static_cast(type_.type_enum_); + } + return *this; +} + +RTAnyType RTAny::type() const { return type_; } + +RTAny RTAny::from_vertex(label_t l, vid_t v) { + RTAny ret; + ret.type_ = RTAnyType::kVertex; + ret.value_.vertex.first = l; + ret.value_.vertex.second = v; + return ret; +} + +RTAny RTAny::from_vertex(const std::pair& v) { + RTAny ret; + ret.type_ = RTAnyType::kVertex; + ret.value_.vertex = v; + return ret; +} + +RTAny RTAny::from_edge( + const std::tuple& v) { + RTAny ret; + ret.type_ = RTAnyType::kEdge; + ret.value_.edge = v; + return ret; +} + +RTAny RTAny::from_bool(bool v) { + RTAny ret; + ret.type_ = RTAnyType::kBoolValue; + ret.value_.b_val = v; + return ret; +} + +RTAny RTAny::from_int64(int64_t v) { + RTAny ret; + ret.type_ = RTAnyType::kI64Value; + ret.value_.i64_val = v; + return ret; +} + +RTAny RTAny::from_uint64(uint64_t v) { + RTAny ret; + ret.type_ = RTAnyType::kU64Value; + ret.value_.u64_val = v; + return ret; +} + +RTAny RTAny::from_int32(int v) { + RTAny ret; + ret.type_ = RTAnyType::kI32Value; + ret.value_.i32_val = v; + return ret; +} + +RTAny RTAny::from_string(const std::string& str) { + RTAny ret; + ret.type_ = RTAnyType::kStringValue; + ret.value_.str_val = std::string_view(str); + return ret; +} + +RTAny RTAny::from_string(const std::string_view& str) { + RTAny ret; + ret.type_ = RTAnyType::kStringValue; + ret.value_.str_val = str; + return ret; +} + +RTAny RTAny::from_string_set(const std::set& str_set) { + RTAny ret; + ret.type_ = RTAnyType::kStringSetValue; + ret.value_.str_set = &str_set; + return ret; +} + +RTAny RTAny::from_vertex_list(const std::vector& v_set) { + RTAny ret; + ret.type_ = RTAnyType::kVertexSetValue; + ret.value_.vset = &v_set; + return ret; +} + +RTAny RTAny::from_date32(Date v) { + RTAny ret; + ret.type_ = RTAnyType::kDate32; + ret.value_.i64_val = v.milli_second; + return ret; +} + +RTAny RTAny::from_tuple(std::vector&& v) { + RTAny ret; + ret.type_ = RTAnyType::kTuple; + ret.value_.t.init(std::move(v)); + return ret; +} + +RTAny RTAny::from_tuple(const Tuple& t) { + RTAny ret; + ret.type_ = RTAnyType::kTuple; + ret.value_.t = t.dup(); + return ret; +} + +RTAny RTAny::from_list(const List& l) { + RTAny ret; + ret.type_ = RTAnyType::kList; + ret.value_.list = std::move(l); + return ret; +} + +RTAny RTAny::from_double(double v) { + RTAny ret; + ret.type_ = RTAnyType::kF64Value; + ret.value_.f64_val = v; + return ret; +} + +RTAny RTAny::from_map(const Map& m) { + RTAny ret; + ret.type_ = RTAnyType::kMap; + ret.value_.map = std::move(m); + return ret; +} + +bool RTAny::as_bool() const { + if (type_ == RTAnyType::kNull) { + return false; + } + CHECK(type_ == RTAnyType::kBoolValue) + << "type_ = " << static_cast(type_.type_enum_); + return value_.b_val; +} +int RTAny::as_int32() const { + CHECK(type_ == RTAnyType::kI32Value) + << "type_ = " << static_cast(type_.type_enum_); + return value_.i32_val; +} +int64_t RTAny::as_int64() const { + CHECK(type_ == RTAnyType::kI64Value); + return value_.i64_val; +} +uint64_t RTAny::as_uint64() const { + CHECK(type_ == RTAnyType::kU64Value); + return value_.u64_val; +} +int64_t RTAny::as_date32() const { + CHECK(type_ == RTAnyType::kDate32); + return value_.i64_val; +} + +double RTAny::as_double() const { + CHECK(type_ == RTAnyType::kF64Value); + return value_.f64_val; +} + +const std::pair& RTAny::as_vertex() const { + CHECK(type_ == RTAnyType::kVertex); + return value_.vertex; +} +const std::tuple& RTAny::as_edge() + const { + CHECK(type_ == RTAnyType::kEdge); + return value_.edge; +} +const std::set& RTAny::as_string_set() const { + CHECK(type_ == RTAnyType::kStringSetValue); + return *value_.str_set; +} +std::string_view RTAny::as_string() const { + if (type_ == RTAnyType::kStringValue) { + return value_.str_val; + } else if (type_ == RTAnyType::kUnknown) { + return std::string_view(); + } else { + LOG(FATAL) << "unexpected type" << static_cast(type_.type_enum_); + return std::string_view(); + } +} + +List RTAny::as_list() const { + CHECK(type_ == RTAnyType::kList); + return value_.list; +} +const std::vector& RTAny::as_vertex_list() const { + CHECK(type_ == RTAnyType::kVertexSetValue); + return *value_.vset; +} + +Path RTAny::as_path() const { + CHECK(type_ == RTAnyType::kPath); + return value_.p; +} + +Tuple RTAny::as_tuple() const { + CHECK(type_ == RTAnyType::kTuple); + return value_.t; +} + +Map RTAny::as_map() const { + CHECK(type_ == RTAnyType::kMap); + return value_.map; +} + +int RTAny::numerical_cmp(const RTAny& other) const { + switch (type_.type_enum_) { + case RTAnyType::RTAnyTypeImpl::kI64Value: + switch (other.type_.type_enum_) { + case RTAnyType::RTAnyTypeImpl::kI32Value: + return value_.i64_val - other.value_.i32_val; + case RTAnyType::RTAnyTypeImpl::kF64Value: + return value_.i64_val - other.value_.f64_val; + default: + LOG(FATAL) << "not support for " + << static_cast(other.type_.type_enum_); + } + break; + case RTAnyType::RTAnyTypeImpl::kI32Value: + switch (other.type_.type_enum_) { + case RTAnyType::RTAnyTypeImpl::kI64Value: + return value_.i32_val - other.value_.i64_val; + case RTAnyType::RTAnyTypeImpl::kF64Value: + return value_.i32_val - other.value_.f64_val; + default: + LOG(FATAL) << "not support for " + << static_cast(other.type_.type_enum_); + } + break; + case RTAnyType::RTAnyTypeImpl::kF64Value: + switch (other.type_.type_enum_) { + case RTAnyType::RTAnyTypeImpl::kI64Value: + return value_.f64_val - other.value_.i64_val; + case RTAnyType::RTAnyTypeImpl::kI32Value: + return value_.f64_val - other.value_.i32_val; + default: + LOG(FATAL) << "not support for " << static_cast(type_.type_enum_); + } + break; + default: + LOG(FATAL) << "not support for " << static_cast(type_.type_enum_); + } +} +inline static bool is_numerical_type(const RTAnyType& type) { + return type == RTAnyType::kI64Value || type == RTAnyType::kI32Value || + type == RTAnyType::kF64Value; +} + +bool RTAny::operator<(const RTAny& other) const { + if (type_.type_enum_ != other.type_.type_enum_) { + if (is_numerical_type(type_) && is_numerical_type(other.type_)) { + return numerical_cmp(other) < 0; + } else { + return false; + } + } + if (type_ == RTAnyType::kI64Value) { + return value_.i64_val < other.value_.i64_val; + + } else if (type_ == RTAnyType::kI32Value) { + return value_.i32_val < other.value_.i32_val; + + } else if (type_ == RTAnyType::kStringValue) { + return value_.str_val < other.value_.str_val; + } else if (type_ == RTAnyType::kDate32) { + return value_.i64_val < other.value_.i64_val; + } else if (type_ == RTAnyType::kF64Value) { + return value_.f64_val < other.value_.f64_val; + } + + LOG(FATAL) << "not support for " << static_cast(type_.type_enum_); + return true; +} + +bool RTAny::operator==(const RTAny& other) const { + // assert(type_ == other.type_); + if (type_.type_enum_ != other.type_.type_enum_) { + if (is_numerical_type(type_) && is_numerical_type(other.type_)) { + return numerical_cmp(other) == 0; + } else { + return false; + } + } + + if (type_ == RTAnyType::kI64Value) { + return value_.i64_val == other.value_.i64_val; + } else if (type_ == RTAnyType::kI32Value) { + return value_.i32_val == other.value_.i32_val; + } else if (type_ == RTAnyType::kStringValue) { + return value_.str_val == other.value_.str_val; + } else if (type_ == RTAnyType::kVertex) { + return value_.vertex == other.value_.vertex; + } else if (type_ == RTAnyType::kDate32) { + return value_.i64_val == other.value_.i64_val; + } + + if (type_ == RTAnyType::kI64Value && other.type_ == RTAnyType::kI32Value) { + return value_.i64_val == other.value_.i32_val; + } else if (type_ == RTAnyType::kI32Value && + other.type_ == RTAnyType::kI64Value) { + return value_.i32_val == other.value_.i64_val; + } else if (type_ == RTAnyType::kF64Value) { + return value_.f64_val == other.value_.f64_val; + } + + LOG(FATAL) << "not support..." << static_cast(type_.type_enum_); + return true; +} + +RTAny RTAny::operator+(const RTAny& other) const { + // CHECK(type_ == other.type_); + if (type_ == RTAnyType::kI64Value) { + return RTAny::from_int64(value_.i64_val + other.value_.i64_val); + } + + if (type_ == RTAnyType::kI64Value && other.type_ == RTAnyType::kI32Value) { + return RTAny::from_int64(value_.i64_val + other.value_.i32_val); + } else if (type_ == RTAnyType::kI32Value && + other.type_ == RTAnyType::kI64Value) { + return RTAny::from_int64(value_.i32_val * 1l + other.value_.i64_val); + } else if (type_ == RTAnyType::kF64Value) { + return RTAny::from_double(value_.f64_val + other.value_.f64_val); + } + + LOG(FATAL) << "not support"; + return RTAny(); +} + +RTAny RTAny::operator-(const RTAny& other) const { + // CHECK(type_ == other.type_); + if (type_ == RTAnyType::kI64Value) { + return RTAny::from_int64(value_.i64_val - other.value_.i64_val); + } + + if (type_ == RTAnyType::kI64Value && other.type_ == RTAnyType::kI32Value) { + return RTAny::from_int64(value_.i64_val - other.value_.i32_val); + } else if (type_ == RTAnyType::kI32Value && + other.type_ == RTAnyType::kI64Value) { + return RTAny::from_int64(value_.i32_val * 1l - other.value_.i64_val); + } else if (type_ == RTAnyType::kF64Value) { + return RTAny::from_double(value_.f64_val - other.value_.f64_val); + } + LOG(FATAL) << "not support"; + return RTAny(); +} + +RTAny RTAny::operator/(const RTAny& other) const { + // CHECK(type_ == other.type_); + if (type_ == RTAnyType::kI64Value) { + return RTAny::from_int64(value_.i64_val / other.value_.i64_val); + } + + if (type_ == RTAnyType::kI64Value && other.type_ == RTAnyType::kI32Value) { + return RTAny::from_int64(value_.i64_val / other.value_.i32_val); + } else if (type_ == RTAnyType::kI32Value && + other.type_ == RTAnyType::kI64Value) { + return RTAny::from_int64(value_.i32_val * 1l / other.value_.i64_val); + } + LOG(FATAL) << "not support"; + return RTAny(); +} + +void RTAny::sink_impl(common::Value* value) const { + if (type_ == RTAnyType::kI64Value) { + value->set_i64(value_.i64_val); + } else if (type_ == RTAnyType::kStringValue) { + value->set_str(value_.str_val.data(), value_.str_val.size()); + } else if (type_ == RTAnyType::kI32Value) { + value->set_i32(value_.i32_val); + } else if (type_ == RTAnyType::kStringSetValue) { + LOG(FATAL) << "not support string set sink"; + } else if (type_ == RTAnyType::kDate32) { + value->set_i64(value_.i64_val); + } else if (type_ == RTAnyType::kBoolValue) { + value->set_boolean(value_.b_val); + } else if (type_ == RTAnyType::kF64Value) { + value->set_f64(value_.f64_val); + } else if (type_ == RTAnyType::kList) { + LOG(FATAL) << "not support list sink"; + } else { + LOG(FATAL) << "not implemented for " << static_cast(type_.type_enum_); + } +} + +static void sink_any(const Any& any, common::Value* value) { + if (any.type == PropertyType::Int64()) { + value->set_i64(any.AsInt64()); + } else if (any.type == PropertyType::StringView()) { + auto str = any.AsStringView(); + value->set_str(str.data(), str.size()); + } else if (any.type == PropertyType::Date()) { + value->set_i64(any.AsDate().milli_second); + } else if (any.type == PropertyType::Int32()) { + value->set_i32(any.AsInt32()); + } else if (any.type == PropertyType::Double()) { + value->set_f64(any.AsDouble()); + } else if (any.type == PropertyType::Bool()) { + value->set_boolean(any.AsBool()); + } else if (any.type == PropertyType::Double()) { + value->set_f64(any.AsDouble()); + } else { + LOG(FATAL) << "Any value: " << any.to_string() + << ", type = " << any.type.type_enum; + } +} + +void sink_vertex(const gs::ReadTransaction& txn, + const std::pair& vertex, results::Vertex* v) { + v->mutable_label()->set_id(vertex.first); + v->set_id(encode_unique_vertex_id(vertex.first, vertex.second)); + // TODO: add properties + const auto& names = + txn.graph().schema().get_vertex_property_names(vertex.first); + for (size_t i = 0; i < names.size(); ++i) { + auto prop = v->add_properties(); + prop->mutable_key()->set_name(names[i]); + sink_any(txn.graph().get_vertex_table(vertex.first).at(vertex.second, i), + prop->mutable_value()); + } +} + +void RTAny::sink(const gs::ReadTransaction& txn, int id, + results::Column* col) const { + col->mutable_name_or_id()->set_id(id); + if (type_ == RTAnyType::kList) { + auto collection = col->mutable_entry()->mutable_collection(); + for (size_t i = 0; i < value_.list.size(); ++i) { + value_.list.get(i).sink_impl( + collection->add_collection()->mutable_object()); + } + } else if (type_ == RTAnyType::kStringSetValue) { + auto collection = col->mutable_entry()->mutable_collection(); + for (auto& s : *value_.str_set) { + collection->add_collection()->mutable_object()->set_str(s); + } + } else if (type_ == RTAnyType::kTuple) { + auto collection = col->mutable_entry()->mutable_collection(); + for (size_t i = 0; i < value_.t.size(); ++i) { + value_.t.get(i).sink_impl(collection->add_collection()->mutable_object()); + } + } else if (type_ == RTAnyType::kVertex) { + auto v = col->mutable_entry()->mutable_element()->mutable_vertex(); + sink_vertex(txn, value_.vertex, v); + + } else if (type_ == RTAnyType::kMap) { + auto mp = col->mutable_entry()->mutable_map(); + auto [keys_ptr, vals_ptr] = value_.map.key_vals(); + auto& keys = *keys_ptr; + auto& vals = *vals_ptr; + for (size_t i = 0; i < keys.size(); ++i) { + if (vals[i].is_null()) { + continue; + } + auto ret = mp->add_key_values(); + ret->mutable_key()->set_str(keys[i]); + if (vals[i].type_ == RTAnyType::kVertex) { + auto v = ret->mutable_value()->mutable_element()->mutable_vertex(); + sink_vertex(txn, vals[i].as_vertex(), v); + } else { + vals[i].sink_impl( + ret->mutable_value()->mutable_element()->mutable_object()); + } + } + + } else if (type_ == RTAnyType::kEdge) { + auto e = col->mutable_entry()->mutable_element()->mutable_edge(); + auto [label, src, dst, prop, dir] = this->as_edge(); + e->mutable_src_label()->set_id(label.src_label); + e->mutable_dst_label()->set_id(label.dst_label); + auto edge_label = generate_edge_label_id(label.src_label, label.dst_label, + label.edge_label); + e->mutable_label()->set_id(label.edge_label); + e->set_src_id(encode_unique_vertex_id(label.src_label, src)); + e->set_dst_id(encode_unique_vertex_id(label.dst_label, dst)); + e->set_id(encode_unique_edge_id(edge_label, src, dst)); + auto& prop_names = txn.schema().get_edge_property_names( + label.src_label, label.dst_label, label.edge_label); + auto props = e->add_properties(); + props->mutable_key()->set_name(prop_names[0]); + sink_any(prop, e->mutable_properties(0)->mutable_value()); + } else if (type_ == RTAnyType::kPath) { + LOG(FATAL) << "not support path sink"; + + } else { + sink_impl(col->mutable_entry()->mutable_element()->mutable_object()); + } +} + +void RTAny::encode_sig(RTAnyType type, Encoder& encoder) const { + if (type == RTAnyType::kI64Value) { + encoder.put_long(this->as_int64()); + } else if (type == RTAnyType::kStringValue) { + encoder.put_string_view(this->as_string()); + } else if (type == RTAnyType::kI32Value) { + encoder.put_int(this->as_int32()); + } else if (type == RTAnyType::kVertex) { + const auto& v = this->value_.vertex; + encoder.put_byte(v.first); + encoder.put_int(v.second); + } else if (type == RTAnyType::kEdge) { + const auto& [label, src, dst, prop, dir] = this->as_edge(); + + encoder.put_byte(label.src_label); + encoder.put_byte(label.dst_label); + encoder.put_byte(label.edge_label); + encoder.put_int(src); + encoder.put_int(dst); + encoder.put_byte(dir == Direction::kOut ? 1 : 0); + + } else if (type == RTAnyType::kBoolValue) { + encoder.put_byte(this->as_bool() ? 1 : 0); + } else if (type == RTAnyType::kList) { + encoder.put_int(this->as_list().size()); + List list = this->as_list(); + for (size_t i = 0; i < list.size(); ++i) { + list.get(i).encode_sig(list.get(i).type(), encoder); + } + } else if (type == RTAnyType::kTuple) { + Tuple tuple = this->as_tuple(); + encoder.put_int(tuple.size()); + for (size_t i = 0; i < tuple.size(); ++i) { + tuple.get(i).encode_sig(tuple.get(i).type(), encoder); + } + } else if (type == RTAnyType::kNull) { + encoder.put_int(-1); + } else if (type == RTAnyType::kF64Value) { + encoder.put_double(this->as_double()); + } else { + LOG(FATAL) << "not implemented for " << static_cast(type_.type_enum_); + } +} + +std::string RTAny::to_string() const { + if (type_ == RTAnyType::kI64Value) { + return std::to_string(value_.i64_val); + } else if (type_ == RTAnyType::kStringValue) { + return std::string(value_.str_val); + } else if (type_ == RTAnyType::kI32Value) { + return std::to_string(value_.i32_val); + } else if (type_ == RTAnyType::kVertex) { +#if 0 + return std::string("v") + + std::to_string(static_cast(value_.vertex.first)) + "-" + + std::to_string(value_.vertex.second); +#else + return std::to_string(value_.vertex.second); +#endif + } else if (type_ == RTAnyType::kStringSetValue) { + std::string ret = "{"; + for (auto& str : *value_.str_set) { + ret += str; + ret += ", "; + } + ret += "}"; + return ret; + } else if (type_ == RTAnyType::kEdge) { + auto [label, src, dst, prop, dir] = value_.edge; + return std::to_string(src) + " -> " + std::to_string(dst); + } else if (type_ == RTAnyType::kPath) { + return value_.p.to_string(); + } else if (type_ == RTAnyType::kBoolValue) { + return value_.b_val ? "true" : "false"; + } else if (type_ == RTAnyType::kDate32) { + return std::to_string(value_.i64_val); + } else if (type_ == RTAnyType::kList) { + std::string ret = "["; + for (size_t i = 0; i < value_.list.size(); ++i) { + ret += value_.list.get(i).to_string(); + if (i != value_.list.size() - 1) { + ret += ", "; + } + } + ret += "]"; + return ret; + } else if (type_ == RTAnyType::kTuple) { + std::string ret = "("; + for (size_t i = 0; i < value_.t.size(); ++i) { + ret += value_.t.get(i).to_string(); + if (i != value_.t.size() - 1) { + ret += ", "; + } + } + ret += ")"; + return ret; + } else if (type_ == RTAnyType::kNull) { + return "null"; + } else if (type_ == RTAnyType::kF64Value) { + return std::to_string(value_.f64_val); + } else { + LOG(FATAL) << "not implemented for " << static_cast(type_.type_enum_); + return ""; + } +} + +} // namespace runtime + +} // namespace gs diff --git a/flex/engines/graph_db/runtime/common/rt_any.h b/flex/engines/graph_db/runtime/common/rt_any.h new file mode 100644 index 000000000000..509f9e60bbb9 --- /dev/null +++ b/flex/engines/graph_db/runtime/common/rt_any.h @@ -0,0 +1,464 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RUNTIME_COMMON_RT_ANY_H_ +#define RUNTIME_COMMON_RT_ANY_H_ + +#include "flex/proto_generated_gie/results.pb.h" +#include "flex/proto_generated_gie/type.pb.h" + +#include "flex/engines/graph_db/database/read_transaction.h" +#include "flex/engines/graph_db/runtime/common/types.h" +#include "flex/utils/app_utils.h" + +namespace gs { + +namespace runtime { + +class PathImpl { + public: + static std::shared_ptr make_path_impl(label_t label, vid_t v) { + auto new_path = std::make_shared(); + new_path->path_.push_back(std::make_pair(label, v)); + return new_path; + } + std::shared_ptr expand(label_t label, vid_t v) const { + auto new_path = std::make_shared(); + new_path->path_ = path_; + new_path->path_.push_back(std::make_pair(label, v)); + return new_path; + } + + std::string to_string() const { + std::string str; + for (size_t i = 0; i < path_.size(); ++i) { + str += "(" + std::to_string(static_cast(path_[i].first)) + ", " + + std::to_string(path_[i].second) + ")"; + if (i != path_.size() - 1) { + str += "->"; + } + } + return str; + } + + std::pair get_end() const { return path_.back(); } + std::pair get_start() const { return path_.front(); } + bool operator<(const PathImpl& p) const { return path_ < p.path_; } + bool operator==(const PathImpl& p) const { return path_ == p.path_; } + std::vector> path_; +}; +class Path { + public: + static Path make_path(const std::shared_ptr& impl) { + Path new_path; + new_path.impl_ = impl.get(); + return new_path; + } + + std::string to_string() const { return impl_->to_string(); } + + int32_t len() const { return impl_->path_.size(); } + std::pair get_end() const { return impl_->get_end(); } + + std::pair get_start() const { return impl_->get_start(); } + bool operator<(const Path& p) const { return *impl_ < *(p.impl_); } + bool operator==(const Path& p) const { return *(impl_) == *(p.impl_); } + + PathImpl* impl_; +}; +class RTAny; + +class ListImplBase { + public: + virtual ~ListImplBase() = default; + virtual bool operator<(const ListImplBase& p) const = 0; + virtual bool operator==(const ListImplBase& p) const = 0; + virtual size_t size() const = 0; + virtual RTAny get(size_t idx) const = 0; +}; + +class List { + public: + static List make_list(const std::shared_ptr& impl) { + List new_list; + new_list.impl_ = impl.get(); + return new_list; + } + + bool operator<(const List& p) const { return *impl_ < *(p.impl_); } + bool operator==(const List& p) const { return *(impl_) == *(p.impl_); } + size_t size() const { return impl_->size(); } + RTAny get(size_t idx) const; + + ListImplBase* impl_; +}; + +class Tuple { + public: + ~Tuple() { + if (props_ != nullptr) { + // delete props_; + } + } + void init(std::vector&& vals) { + props_ = new std::vector(vals); + } + // not support for path + Tuple dup() const { + Tuple new_tuple; + new_tuple.props_ = new std::vector(*props_); + return new_tuple; + } + + bool operator<(const Tuple& p) const { return *props_ < *(p.props_); } + bool operator==(const Tuple& p) const { return *props_ == *(p.props_); } + size_t size() const { return props_->size(); } + const RTAny& get(size_t idx) const { return (*props_)[idx]; } + + private: + const std::vector* props_; +}; + +class MapImpl { + public: + static MapImpl make_map_impl(const std::vector* keys, + const std::vector* values) { + MapImpl map_impl; + map_impl.keys = keys; + map_impl.values = values; + return map_impl; + } + size_t size() const { return keys->size(); } + + const std::vector* keys; + const std::vector* values; +}; + +class RTAnyType { + public: + enum class RTAnyTypeImpl { + kVertex, + kEdge, + kI64Value, + kU64Value, + kI32Value, + kF64Value, + kBoolValue, + kStringValue, + kVertexSetValue, + kStringSetValue, + kUnknown, + kDate32, + kPath, + kNull, + kTuple, + kList, + kMap, + }; + static const RTAnyType kVertex; + static const RTAnyType kEdge; + static const RTAnyType kI64Value; + static const RTAnyType kU64Value; + static const RTAnyType kI32Value; + static const RTAnyType kF64Value; + static const RTAnyType kBoolValue; + static const RTAnyType kStringValue; + static const RTAnyType kVertexSetValue; + static const RTAnyType kStringSetValue; + static const RTAnyType kUnknown; + static const RTAnyType kDate32; + static const RTAnyType kPath; + static const RTAnyType kNull; + static const RTAnyType kTuple; + static const RTAnyType kList; + static const RTAnyType kMap; + + RTAnyType() : type_enum_(RTAnyTypeImpl::kUnknown) {} + RTAnyType(const RTAnyType& other) + : type_enum_(other.type_enum_), null_able_(other.null_able_) {} + RTAnyType(RTAnyTypeImpl type) : type_enum_(type), null_able_(false) {} + RTAnyType(RTAnyTypeImpl type, bool null_able) + : type_enum_(type), null_able_(null_able) {} + bool operator==(const RTAnyType& other) const { + return type_enum_ == other.type_enum_; + } + RTAnyTypeImpl type_enum_; + bool null_able_; +}; + +class Map { + public: + static Map make_map(MapImpl impl) { + Map m; + m.map_ = impl; + return m; + } + std::pair*, const std::vector*> + key_vals() const { + return std::make_pair(map_.keys, map_.values); + } + + MapImpl map_; +}; + +RTAnyType parse_from_ir_data_type(const ::common::IrDataType& dt); + +union RTAnyValue { + RTAnyValue() : vset(NULL) {} + ~RTAnyValue() {} + + std::pair vertex; + std::tuple edge; + int64_t i64_val; + uint64_t u64_val; + int i32_val; + double f64_val; + const std::vector* vset; + const std::set* str_set; + std::string_view str_val; + Path p; + Tuple t; + List list; + Map map; + bool b_val; +}; + +class RTAny { + public: + RTAny(); + RTAny(RTAnyType type); + RTAny(const Any& val); + RTAny(const RTAny& rhs); + RTAny(const Path& p); + ~RTAny() = default; + bool is_null() const { return type_ == RTAnyType::kNull; } + + int numerical_cmp(const RTAny& other) const; + + RTAny& operator=(const RTAny& rhs); + + static RTAny from_vertex(label_t l, vid_t v); + static RTAny from_vertex(const std::pair& v); + static RTAny from_edge( + const std::tuple& v); + static RTAny from_bool(bool v); + static RTAny from_int64(int64_t v); + static RTAny from_uint64(uint64_t v); + static RTAny from_int32(int v); + static RTAny from_string(const std::string& str); + static RTAny from_string(const std::string_view& str); + static RTAny from_string_set(const std::set& str_set); + static RTAny from_vertex_list(const std::vector& v_set); + static RTAny from_date32(Date v); + static RTAny from_tuple(std::vector&& tuple); + static RTAny from_tuple(const Tuple& tuple); + static RTAny from_list(const List& list); + static RTAny from_double(double v); + static RTAny from_map(const Map& m); + + bool as_bool() const; + int as_int32() const; + int64_t as_int64() const; + uint64_t as_uint64() const; + int64_t as_date32() const; + double as_double() const; + const std::pair& as_vertex() const; + const std::tuple& as_edge() const; + const std::set& as_string_set() const; + std::string_view as_string() const; + const std::vector& as_vertex_list() const; + Path as_path() const; + Tuple as_tuple() const; + List as_list() const; + Map as_map() const; + + bool operator<(const RTAny& other) const; + bool operator==(const RTAny& other) const; + + RTAny operator+(const RTAny& other) const; + + RTAny operator-(const RTAny& other) const; + RTAny operator/(const RTAny& other) const; + + void sink(const gs::ReadTransaction& txn, int id, + results::Column* column) const; + void encode_sig(RTAnyType type, Encoder& encoder) const; + + std::string to_string() const; + + RTAnyType type() const; + + private: + void sink_impl(common::Value* collection) const; + RTAnyType type_; + RTAnyValue value_; +}; + +template +struct TypedConverter {}; + +template <> +struct TypedConverter { + static RTAnyType type() { return RTAnyType::kBoolValue; } + static bool to_typed(const RTAny& val) { return val.as_bool(); } + static RTAny from_typed(bool val) { return RTAny::from_bool(val); } + static const std::string name() { return "bool"; } +}; +template <> +struct TypedConverter { + static RTAnyType type() { return RTAnyType::kI32Value; } + static int to_typed(const RTAny& val) { return val.as_int32(); } + static RTAny from_typed(int val) { return RTAny::from_int32(val); } + static const std::string name() { return "int"; } +}; + +template <> +struct TypedConverter> { + static RTAnyType type() { return RTAnyType::kStringSetValue; } + static const std::set to_typed(const RTAny& val) { + return val.as_string_set(); + } + static RTAny from_typed(const std::set& val) { + return RTAny::from_string_set(val); + } + static const std::string name() { return "set"; } +}; + +template <> +struct TypedConverter> { + static RTAnyType type() { return RTAnyType::kVertexSetValue; } + static const std::vector& to_typed(const RTAny& val) { + return val.as_vertex_list(); + } + static RTAny from_typed(const std::vector& val) { + return RTAny::from_vertex_list(val); + } + static const std::string name() { return "vector"; } +}; + +template <> +struct TypedConverter { + static RTAnyType type() { return RTAnyType::kStringValue; } + static std::string_view to_typed(const RTAny& val) { return val.as_string(); } + static RTAny from_typed(const std::string_view& val) { + return RTAny::from_string(val); + } + static const std::string name() { return "string_view"; } + static std::string_view typed_from_string(const std::string& str) { + return std::string_view(str.data(), str.size()); + } +}; + +template <> +struct TypedConverter { + static RTAnyType type() { return RTAnyType::kU64Value; } + static uint64_t to_typed(const RTAny& val) { return val.as_uint64(); } + static RTAny from_typed(uint64_t val) { return RTAny::from_uint64(val); } + static const std::string name() { return "uint64"; } +}; + +template <> +struct TypedConverter { + static RTAnyType type() { return RTAnyType::kI64Value; } + static int64_t to_typed(const RTAny& val) { return val.as_int64(); } + static RTAny from_typed(int64_t val) { return RTAny::from_int64(val); } + static const std::string name() { return "int64"; } +}; + +template <> +struct TypedConverter { + static RTAnyType type() { return RTAnyType::kI64Value; } + static double to_typed(const RTAny& val) { return val.as_double(); } + static RTAny from_typed(double val) { return RTAny::from_double(val); } + static const std::string name() { return "int64"; } +}; +template <> +struct TypedConverter { + static RTAnyType type() { return RTAnyType::kDate32; } + static Date to_typed(const RTAny& val) { return val.as_date32(); } + static RTAny from_typed(Date val) { return RTAny::from_date32(val); } + static const std::string name() { return "int64"; } + static Date typed_from_string(const std::string& str) { + int64_t val = std::stoll(str); + return Date(val); + } +}; + +template <> +struct TypedConverter { + static RTAnyType type() { return RTAnyType::kTuple; } + static Tuple to_typed(const RTAny& val) { return val.as_tuple(); } + static RTAny from_typed(Tuple val) { + return RTAny::from_tuple(std::move(val)); + } + static const std::string name() { return "tuple"; } +}; + +template <> +struct TypedConverter { + static RTAnyType type() { return RTAnyType::kMap; } + static Map to_typed(const RTAny& val) { return val.as_map(); } + static RTAny from_typed(Map val) { return RTAny::from_map(val); } + static const std::string name() { return "map"; } +}; +template +class ListImpl : ListImplBase { + public: + ListImpl() = default; + static std::shared_ptr make_list_impl(std::vector&& vals) { + auto new_list = new ListImpl(); + new_list->list_ = std::move(vals); + return std::shared_ptr(static_cast(new_list)); + } + + static std::shared_ptr make_list_impl( + const std::vector& vals) { + auto new_list = new ListImpl(); + for (auto& val : vals) { + if (val.is_null()) { + new_list->is_valid_.push_back(false); + new_list->list_.push_back(T()); + } else { + new_list->list_.push_back(TypedConverter::to_typed(val)); + new_list->is_valid_.push_back(true); + } + } + return std::shared_ptr(static_cast(new_list)); + } + + bool operator<(const ListImplBase& p) const { + return list_ < (dynamic_cast&>(p)).list_; + } + bool operator==(const ListImplBase& p) const { + return list_ == (dynamic_cast&>(p)).list_; + } + size_t size() const { return list_.size(); } + RTAny get(size_t idx) const { + if (is_valid_[idx]) { + return TypedConverter::from_typed(list_[idx]); + } else { + return RTAny(RTAnyType::kNull); + } + } + + std::vector list_; + std::vector is_valid_; +}; + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_COMMON_RT_ANY_H_ \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/types.cc b/flex/engines/graph_db/runtime/common/types.cc new file mode 100644 index 000000000000..4f0bf79a5b98 --- /dev/null +++ b/flex/engines/graph_db/runtime/common/types.cc @@ -0,0 +1,55 @@ + +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "flex/engines/graph_db/runtime/common/types.h" + +namespace gs { +namespace runtime { +uint64_t encode_unique_vertex_id(label_t label_id, vid_t vid) { + // encode label_id and vid to a unique vid + GlobalId global_id(label_id, vid); + return global_id.global_id; +} + +uint32_t generate_edge_label_id(label_t src_label_id, label_t dst_label_id, + label_t edge_label_id) { + uint32_t unique_edge_label_id = src_label_id; + static constexpr int num_bits = sizeof(label_t) * 8; + unique_edge_label_id = unique_edge_label_id << num_bits; + unique_edge_label_id = unique_edge_label_id | dst_label_id; + unique_edge_label_id = unique_edge_label_id << num_bits; + unique_edge_label_id = unique_edge_label_id | edge_label_id; + return unique_edge_label_id; +} + +int64_t encode_unique_edge_id(uint32_t label_id, vid_t src, vid_t dst) { + // We assume label_id is only used by 24 bits. + int64_t unique_edge_id = label_id; + static constexpr int num_bits = sizeof(int64_t) * 8 - sizeof(uint32_t) * 8; + unique_edge_id = unique_edge_id << num_bits; + // bitmask for top 44 bits set to 1 + int64_t bitmask = 0xFFFFFFFFFF000000; + // 24 bit | 20 bit | 20 bit + if (bitmask & (int64_t) src || bitmask & (int64_t) dst) { + LOG(ERROR) << "src or dst is too large to be encoded in 20 bits: " << src + << " " << dst; + } + unique_edge_id = unique_edge_id | (src << 20); + unique_edge_id = unique_edge_id | dst; + return unique_edge_id; +} +} // namespace runtime +} // namespace gs \ No newline at end of file diff --git a/flex/engines/graph_db/runtime/common/types.h b/flex/engines/graph_db/runtime/common/types.h new file mode 100644 index 000000000000..cd4e576733b6 --- /dev/null +++ b/flex/engines/graph_db/runtime/common/types.h @@ -0,0 +1,87 @@ +/** Copyright 2020 Alibaba Group Holding Limited. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef RUNTIME_COMMON_TYPES_H_ +#define RUNTIME_COMMON_TYPES_H_ + +#include + +#include "flex/storages/rt_mutable_graph/types.h" +#include "flex/utils/property/types.h" + +namespace gs { + +namespace runtime { + +uint64_t encode_unique_vertex_id(label_t label_id, vid_t vid); +uint32_t generate_edge_label_id(label_t src_label_id, label_t dst_label_id, + label_t edge_label_id); +int64_t encode_unique_edge_id(uint32_t label_id, vid_t src, vid_t dst); +enum class Direction { + kOut, + kIn, + kBoth, +}; + +enum class VOpt { + kStart, + kEnd, + kOther, + kBoth, + kItself, +}; + +enum class JoinKind { + kSemiJoin, + kInnerJoin, + kAntiJoin, + kLeftOuterJoin, +}; + +struct LabelTriplet { + LabelTriplet(label_t src, label_t dst, label_t edge) + : src_label(src), dst_label(dst), edge_label(edge) {} + + std::string to_string() const { + return "(" + std::to_string(static_cast(src_label)) + "-" + + std::to_string(static_cast(edge_label)) + "-" + + std::to_string(static_cast(dst_label)) + ")"; + } + + bool operator==(const LabelTriplet& rhs) const { + return src_label == rhs.src_label && dst_label == rhs.dst_label && + edge_label == rhs.edge_label; + } + + bool operator<(const LabelTriplet& rhs) const { + if (src_label != rhs.src_label) { + return src_label < rhs.src_label; + } + if (dst_label != rhs.dst_label) { + return dst_label < rhs.dst_label; + } + return edge_label < rhs.edge_label; + } + + label_t src_label; + label_t dst_label; + label_t edge_label; +}; + +} // namespace runtime + +} // namespace gs + +#endif // RUNTIME_COMMON_TYPES_H_ \ No newline at end of file diff --git a/flex/engines/hqps_db/core/base_engine.h b/flex/engines/hqps_db/core/base_engine.h index fd4307c90b10..fd56c4410b8d 100644 --- a/flex/engines/hqps_db/core/base_engine.h +++ b/flex/engines/hqps_db/core/base_engine.h @@ -828,9 +828,11 @@ class BaseEngine { auto& head_y = ctx_y.GetMutableHead(); auto left_repeat_array = ctx_x.ObtainOffsetFromTag(real_alias_x - 1); auto right_repeat_array = ctx_y.ObtainOffsetFromTag(real_alias_y - 1); - CHECK(left_repeat_array.size() == right_repeat_array.size()) - << "left size " << left_repeat_array.size() << " right size " - << right_repeat_array.size(); + if (left_repeat_array.size() != right_repeat_array.size()) { + throw std::runtime_error("The two context has different repeat size.: " + + std::to_string(left_repeat_array.size()) + ", " + + std::to_string(right_repeat_array.size())); + } std::vector active_indices, new_offsets; std::tie(active_indices, new_offsets) = @@ -867,7 +869,11 @@ class BaseEngine { } grape::Bitset bitset; bitset.init(max_vid + 1); - CHECK(left_repeat_array.size() == right_repeat_array.size()); + if (left_repeat_array.size() != right_repeat_array.size()) { + throw std::runtime_error("The two context has different repeat size.: " + + std::to_string(left_repeat_array.size()) + ", " + + std::to_string(right_repeat_array.size())); + } for (size_t i = 0; i + 1 < left_repeat_array.size(); ++i) { auto x_start = left_repeat_array[i]; auto x_end = left_repeat_array[i + 1]; diff --git a/flex/engines/hqps_db/core/operator/get_v.h b/flex/engines/hqps_db/core/operator/get_v.h index e396a20e608b..887d66ee33c5 100644 --- a/flex/engines/hqps_db/core/operator/get_v.h +++ b/flex/engines/hqps_db/core/operator/get_v.h @@ -109,8 +109,10 @@ class GetVertex { const GRAPH_INTERFACE& graph, const SET_T& set, const GetVOpt& get_v_opt) { auto v_opt = get_v_opt.v_opt_; - CHECK(v_opt == VOpt::Itself) - << "Can only get v from vertex set with v_opt == vopt::Itself"; + if (v_opt != VOpt::Itself) { + throw std::runtime_error( + "Can only get v from vertex set with v_opt == vopt::Itself"); + } auto v_labels = get_v_opt.v_labels_; auto props = get_v_opt.props_; auto expr = get_v_opt.expr_; @@ -236,7 +238,9 @@ class GetVertex { std::sort(labels.begin(), labels.end()); labels.erase(std::unique(labels.begin(), labels.end()), labels.end()); // Can only be one label. - CHECK(labels.size() == 1); + if (labels.size() != 1) { + throw std::runtime_error("Path set should have only one label"); + } // if req_labels is empty, then use the label from path set. if (req_label_vec.empty()) { req_label_vec.push_back(labels[0]); diff --git a/flex/engines/hqps_db/core/operator/group_by.h b/flex/engines/hqps_db/core/operator/group_by.h index c2ee6e0df8e6..ff261a3f163e 100644 --- a/flex/engines/hqps_db/core/operator/group_by.h +++ b/flex/engines/hqps_db/core/operator/group_by.h @@ -489,7 +489,6 @@ class GroupByOp { } auto data_ele = gs::get_from_tuple(data_tuple); int32_t ind = insert_to_keyed_set(keyed_set_builder, key_ele, data_ele); - // CHECK(ind != -1); if (ind != -1) { insert_to_value_set_builder(value_set_builder_tuple, ele_tuple, data_tuple, ind); diff --git a/flex/engines/http_server/handler/graph_db_http_handler.cc b/flex/engines/http_server/handler/graph_db_http_handler.cc index eb09419bb025..56020f9538b6 100644 --- a/flex/engines/http_server/handler/graph_db_http_handler.cc +++ b/flex/engines/http_server/handler/graph_db_http_handler.cc @@ -444,6 +444,177 @@ class stored_proc_handler : public StoppableHandler { #endif }; +class adhoc_runtime_query_handler : public StoppableHandler { + public: + static std::vector>& get_executors() { + static std::vector> executor_refs; + return executor_refs; + } + + adhoc_runtime_query_handler(uint32_t init_group_id, uint32_t max_group_id, + uint32_t group_inc_step, + uint32_t shard_concurrency) + : StoppableHandler(init_group_id, max_group_id, group_inc_step, + shard_concurrency), + executor_idx_(0) { + auto& executor_refs = get_executors(); + CHECK(executor_refs.size() >= StoppableHandler::shard_id()); + executor_refs[StoppableHandler::shard_id()].reserve(shard_concurrency_); + { + hiactor::scope_builder builder; + builder.set_shard(StoppableHandler::shard_id()) + .enter_sub_scope(hiactor::scope(0)) + .enter_sub_scope(hiactor::scope(init_group_id)); + for (unsigned i = 0; i < shard_concurrency_; ++i) { + executor_refs[StoppableHandler::shard_id()].emplace_back( + builder.build_ref(i)); + } + } +#ifdef HAVE_OPENTELEMETRY_CPP + total_counter_ = otel::create_int_counter("hqps_adhoc_query_total"); + latency_histogram_ = + otel::create_double_histogram("hqps_adhoc_query_latency"); +#endif // HAVE_OPENTELEMETRY_CPP + } + + ~adhoc_runtime_query_handler() override = default; + + seastar::future<> stop() override { + return StoppableHandler::cancel_scope([this] { + LOG(INFO) << "Stopping adhoc actors on shard id: " + << StoppableHandler::shard_id(); + get_executors()[StoppableHandler::shard_id()].clear(); + }); + } + + bool start() override { + if (get_executors()[StoppableHandler::shard_id()].size() > 0) { + LOG(ERROR) << "The actors have been already created!"; + return false; + } + return StoppableHandler::start_scope( + [this](hiactor::scope_builder& builder) { + for (unsigned i = 0; i < StoppableHandler::shard_concurrency_; ++i) { + get_executors()[StoppableHandler::shard_id()].emplace_back( + builder.build_ref(i)); + } + }); + } + + seastar::future> handle( + const seastar::sstring& path, + std::unique_ptr req, + std::unique_ptr rep) override { + auto dst_executor = executor_idx_; + executor_idx_ = (executor_idx_ + 1) % shard_concurrency_; + + if (path != "/v1/graph/current/adhoc_query" && + req->param.exists("graph_id")) { + // TODO(zhanglei): get from graph_db. + if (!is_running_graph(req->param["graph_id"])) { + rep->set_status( + seastar::httpd::reply::status_type::internal_server_error); + rep->write_body("bin", + seastar::sstring("The querying query is not running:" + + req->param["graph_id"])); + rep->done(); + return seastar::make_ready_future< + std::unique_ptr>(std::move(rep)); + } + } + +#ifdef HAVE_OPENTELEMETRY_CPP + auto tracer = otel::get_tracer("adhoc_runtime_query_handler"); + // Extract context from headers. This copy is necessary to avoid access + // after header content been freed + std::map headers(req->_headers.begin(), + req->_headers.end()); + auto current_ctx = opentelemetry::context::RuntimeContext::GetCurrent(); + auto options = otel::get_parent_ctx(current_ctx, headers); + auto outer_span = tracer->StartSpan("adhoc_query_handling", options); + auto scope = tracer->WithActiveSpan(outer_span); + // create a new span for query execution, not started. + auto start_ts = std::chrono::duration_cast( + std::chrono::system_clock::now().time_since_epoch()) + .count(); +#endif // HAVE_OPENTELEMETRY_CPP + +#ifdef HAVE_OPENTELEMETRY_CPP + options.parent = outer_span->GetContext(); + auto query_span = tracer->StartSpan("adhoc_query_execution", options); + auto query_scope = tracer->WithActiveSpan(query_span); +#endif // HAVE_OPENTELEMETRY_CPP + // TODO(zhanglei): choose read or write based on the request, after the + // read/write info is supported in physical plan + // The content contains the path to dynamic library + req->content.append(gs::Schema::ADHOC_READ_PLUGIN_ID_STR, 1); + req->content.append(gs::GraphDBSession::kCypherProtoAdhocStr, 1); + return get_executors()[StoppableHandler::shard_id()][dst_executor] + .run_graph_db_query(query_param{std::move(req->content)}) + .then([ +#ifdef HAVE_OPENTELEMETRY_CPP + query_span = query_span, query_scope = std::move(query_scope) +#endif // HAVE_OPENTELEMETRY_CPP + ](auto&& output) { + return seastar::make_ready_future( + std::move(output.content)); + }) + .then_wrapped([rep = std::move(rep) +#ifdef HAVE_OPENTELEMETRY_CPP + , + this, outer_span, start_ts +#endif // HAVE_OPENTELEMETRY_CPP + ](seastar::future&& fut) mutable { + if (__builtin_expect(fut.failed(), false)) { + rep->set_status( + seastar::httpd::reply::status_type::internal_server_error); + try { + std::rethrow_exception(fut.get_exception()); + } catch (std::exception& e) { + rep->write_body("bin", seastar::sstring(e.what())); + } +#ifdef HAVE_OPENTELEMETRY_CPP + outer_span->SetStatus(opentelemetry::trace::StatusCode::kError, + "Internal Server Error"); + outer_span->End(); + std::map labels = {{ "status", "fail" }}; + total_counter_->Add(1, labels); +#endif // HAVE_OPENTELEMETRY_CPP + rep->done(); + return seastar::make_ready_future< + std::unique_ptr>(std::move(rep)); + } + auto result = fut.get0(); + rep->write_body("bin", std::move(result.content)); +#ifdef HAVE_OPENTELEMETRY_CPP + outer_span->End(); + std::map labels = {{ "status", "success" }}; + total_counter_->Add(1, labels); + auto end_ts = gs::GetCurrentTimeStamp(); +#if OPENTELEMETRY_ABI_VERSION_NO >= 2 + latency_histogram_->Record(end_ts - start_ts); +#else + latency_histogram_->Record(end_ts - start_ts, + opentelemetry::context::Context{}); +#endif +#endif // HAVE_OPENTELEMETRY_CPP + rep->done(); + return seastar::make_ready_future< + std::unique_ptr>(std::move(rep)); + }); + } + + private: + uint32_t executor_idx_; + +#ifdef HAVE_OPENTELEMETRY_CPP + opentelemetry::nostd::unique_ptr> + total_counter_; + opentelemetry::nostd::unique_ptr> + latency_histogram_; +#endif +}; + class adhoc_query_handler : public StoppableHandler { public: static std::vector>& get_executors() { @@ -692,12 +863,14 @@ graph_db_http_handler::graph_db_http_handler(uint16_t http_port, actors_running_(true) { current_graph_query_handlers_.resize(shard_num); all_graph_query_handlers_.resize(shard_num); + adhoc_query_handlers_.resize(shard_num); vertex_handlers_.resize(shard_num); edge_handlers_.resize(shard_num); if (enable_adhoc_handlers_) { - adhoc_query_handlers_.resize(shard_num); adhoc_query_handler::get_executors().resize(shard_num); adhoc_query_handler::get_codegen_actors().resize(shard_num); + } else { + adhoc_runtime_query_handler::get_executors().resize(shard_num); } stored_proc_handler::get_executors().resize(shard_num); } @@ -831,6 +1004,19 @@ seastar::future<> graph_db_http_handler::set_routes() { .add_matcher(new seastar::httpd::optional_param_matcher("graph_id")) .add_str("/adhoc_query"); r.add(rule_adhoc, seastar::httpd::operation_type::POST); + } else { + auto adhoc_runtime_query_handler_ = new adhoc_runtime_query_handler( + ic_adhoc_group_id, max_group_id, group_inc_step, + shard_adhoc_concurrency); + adhoc_query_handlers_[hiactor::local_shard_id()] = + adhoc_runtime_query_handler_; + // Add routes + auto rule_adhoc = + new seastar::httpd::match_rule(adhoc_runtime_query_handler_); + rule_adhoc->add_str("/v1/graph") + .add_matcher(new seastar::httpd::optional_param_matcher("graph_id")) + .add_str("/adhoc_query"); + r.add(rule_adhoc, seastar::httpd::operation_type::POST); } for (size_t i = 0; i < NUM_OPERATION; ++i) { vertex_handlers_[hiactor::local_shard_id()][i] = diff --git a/flex/storages/rt_mutable_graph/schema.cc b/flex/storages/rt_mutable_graph/schema.cc index 4e760b0e2ac5..4326858a4836 100644 --- a/flex/storages/rt_mutable_graph/schema.cc +++ b/flex/storages/rt_mutable_graph/schema.cc @@ -19,6 +19,11 @@ namespace gs { +#define THROW_EXCEPTION_IF(cond, msg) \ + if (cond) { \ + throw std::runtime_error(msg); \ + } + Schema::Schema() = default; Schema::~Schema() = default; @@ -98,11 +103,9 @@ bool Schema::contains_vertex_label(const std::string& label) const { label_t Schema::get_vertex_label_id(const std::string& label) const { label_t ret; - if (vlabel_indexer_.get_index(label, ret)) { - return ret; - } else { - throw std::runtime_error("Fail to get vertex label: " + label); - } + THROW_EXCEPTION_IF(!vlabel_indexer_.get_index(label, ret), + "Fail to get vertex label: " + label); + return ret; } void Schema::set_vertex_properties( @@ -115,8 +118,7 @@ void Schema::set_vertex_properties( const std::vector& Schema::get_vertex_properties( const std::string& label) const { - label_t index; - CHECK(vlabel_indexer_.get_index(label, index)); + label_t index = get_vertex_label_id(label); return vproperties_[index]; } @@ -127,53 +129,56 @@ const std::vector& Schema::get_vertex_properties( const std::vector& Schema::get_vertex_property_names( const std::string& label) const { - label_t index; - CHECK(vlabel_indexer_.get_index(label, index)); - CHECK(index < vprop_names_.size()); - return vprop_names_[index]; + label_t index = get_vertex_label_id(label); + return get_vertex_property_names(index); } const std::vector& Schema::get_vertex_property_names( label_t label) const { - if (label < vprop_names_.size()) { - return vprop_names_[label]; - } else { - throw std::runtime_error("Fail to get vertex property names: " + - std::to_string(label)); - } + THROW_EXCEPTION_IF( + label >= vprop_names_.size(), + "Fail to get vertex property names: " + std::to_string(label) + + ", out of range of vprop_names_ " + + std::to_string(vprop_names_.size())); + return vprop_names_[label]; } const std::string& Schema::get_vertex_description( const std::string& label) const { - label_t index; - CHECK(vlabel_indexer_.get_index(label, index)); + label_t index = get_vertex_label_id(label); return get_vertex_description(index); } const std::string& Schema::get_vertex_description(label_t label) const { - CHECK(label < v_descriptions_.size()); + THROW_EXCEPTION_IF( + label >= v_descriptions_.size(), + "Fail to get vertex description: " + std::to_string(label) + + ", out of range of v_descriptions_ " + + std::to_string(v_descriptions_.size())); return v_descriptions_[label]; } const std::vector& Schema::get_vertex_storage_strategies( const std::string& label) const { - label_t index; - CHECK(vlabel_indexer_.get_index(label, index)); + label_t index = get_vertex_label_id(label); + THROW_EXCEPTION_IF( + index >= vprop_storage_.size(), + "Fail to get vertex storage strategies: " + std::to_string(index) + + ", out of range of vprop_storage_ " + + std::to_string(vprop_storage_.size())); return vprop_storage_[index]; } size_t Schema::get_max_vnum(const std::string& label) const { - label_t index; - CHECK(vlabel_indexer_.get_index(label, index)); + label_t index = get_vertex_label_id(label); return max_vnum_[index]; } bool Schema::exist(const std::string& src_label, const std::string& dst_label, const std::string& edge_label) const { - label_t src, dst, edge; - CHECK(vlabel_indexer_.get_index(src_label, src)); - CHECK(vlabel_indexer_.get_index(dst_label, dst)); - CHECK(elabel_indexer_.get_index(edge_label, edge)); + label_t src = get_vertex_label_id(src_label); + label_t dst = get_vertex_label_id(dst_label); + label_t edge = get_edge_label_id(edge_label); uint32_t index = generate_edge_label(src, dst, edge); return eproperties_.find(index) != eproperties_.end(); } @@ -187,19 +192,23 @@ bool Schema::exist(label_t src_label, label_t dst_label, const std::vector& Schema::get_edge_properties( const std::string& src_label, const std::string& dst_label, const std::string& label) const { - label_t src, dst, edge; - CHECK(vlabel_indexer_.get_index(src_label, src)); - CHECK(vlabel_indexer_.get_index(dst_label, dst)); - CHECK(elabel_indexer_.get_index(label, edge)); + label_t src = get_vertex_label_id(src_label); + label_t dst = get_vertex_label_id(dst_label); + label_t edge = get_edge_label_id(label); uint32_t index = generate_edge_label(src, dst, edge); return eproperties_.at(index); } const std::vector& Schema::get_edge_properties( label_t src_label, label_t dst_label, label_t label) const { - CHECK(src_label < vlabel_indexer_.size()); - CHECK(dst_label < vlabel_indexer_.size()); - CHECK(label < elabel_indexer_.size()); + THROW_EXCEPTION_IF( + src_label >= vlabel_indexer_.size(), + "vertex label " + std::to_string(src_label) + " not found"); + THROW_EXCEPTION_IF( + dst_label >= vlabel_indexer_.size(), + "vertex label " + std::to_string(dst_label) + " not found"); + THROW_EXCEPTION_IF(label >= elabel_indexer_.size(), + "edge label " + std::to_string(label) + " not found"); uint32_t index = generate_edge_label(src_label, dst_label, label); return eproperties_.at(index); } @@ -207,17 +216,27 @@ const std::vector& Schema::get_edge_properties( std::string Schema::get_edge_description(const std::string& src_label, const std::string& dst_label, const std::string& label) const { - label_t src, dst, edge; - CHECK(vlabel_indexer_.get_index(src_label, src)); - CHECK(vlabel_indexer_.get_index(dst_label, dst)); - CHECK(elabel_indexer_.get_index(label, edge)); + label_t src = get_vertex_label_id(src_label); + label_t dst = get_vertex_label_id(dst_label); + label_t edge = get_edge_label_id(label); return get_edge_description(src, dst, edge); } std::string Schema::get_edge_description(label_t src_label, label_t dst_label, label_t label) const { + THROW_EXCEPTION_IF( + src_label >= vlabel_indexer_.size(), + "vertex label " + std::to_string(src_label) + " not found"); + THROW_EXCEPTION_IF( + dst_label >= vlabel_indexer_.size(), + "vertex label " + std::to_string(dst_label) + " not found"); + THROW_EXCEPTION_IF(label >= elabel_indexer_.size(), + "edge label " + std::to_string(label) + " not found"); uint32_t index = generate_edge_label(src_label, dst_label, label); - CHECK(index < e_descriptions_.size()); + THROW_EXCEPTION_IF(index >= e_descriptions_.size(), + "Fail to get edge description: " + std::to_string(index) + + ", out of range of e_descriptions_ " + + std::to_string(e_descriptions_.size())); return e_descriptions_.at(index); } @@ -230,20 +249,23 @@ PropertyType Schema::get_edge_property(label_t src, label_t dst, const std::vector& Schema::get_edge_property_names( const std::string& src_label, const std::string& dst_label, const std::string& label) const { - label_t src, dst, edge; - CHECK(vlabel_indexer_.get_index(src_label, src)); - CHECK(vlabel_indexer_.get_index(dst_label, dst)); - CHECK(elabel_indexer_.get_index(label, edge)); - uint32_t index = generate_edge_label(src, dst, edge); - return eprop_names_.at(index); + label_t src = get_vertex_label_id(src_label); + label_t dst = get_vertex_label_id(dst_label); + label_t edge = get_edge_label_id(label); + return get_edge_property_names(src, dst, edge); } const std::vector& Schema::get_edge_property_names( const label_t& src_label, const label_t& dst_label, const label_t& label) const { - CHECK(src_label < vlabel_indexer_.size()); - CHECK(dst_label < vlabel_indexer_.size()); - CHECK(label < elabel_indexer_.size()); + THROW_EXCEPTION_IF( + src_label >= vlabel_indexer_.size(), + "vertex label " + std::to_string(src_label) + " not found"); + THROW_EXCEPTION_IF( + dst_label >= vlabel_indexer_.size(), + "vertex label " + std::to_string(dst_label) + " not found"); + THROW_EXCEPTION_IF(label >= elabel_indexer_.size(), + "edge label " + std::to_string(label) + " not found"); uint32_t index = generate_edge_label(src_label, dst_label, label); return eprop_names_.at(index); } @@ -251,10 +273,9 @@ const std::vector& Schema::get_edge_property_names( bool Schema::valid_edge_property(const std::string& src_label, const std::string& dst_label, const std::string& label) const { - label_t src, dst, edge; - CHECK(vlabel_indexer_.get_index(src_label, src)); - CHECK(vlabel_indexer_.get_index(dst_label, dst)); - CHECK(elabel_indexer_.get_index(label, edge)); + label_t src = get_vertex_label_id(src_label); + label_t dst = get_vertex_label_id(dst_label); + label_t edge = get_edge_label_id(label); uint32_t index = generate_edge_label(src, dst, edge); return eproperties_.find(index) != eproperties_.end(); } @@ -262,10 +283,9 @@ bool Schema::valid_edge_property(const std::string& src_label, EdgeStrategy Schema::get_outgoing_edge_strategy( const std::string& src_label, const std::string& dst_label, const std::string& label) const { - label_t src, dst, edge; - CHECK(vlabel_indexer_.get_index(src_label, src)); - CHECK(vlabel_indexer_.get_index(dst_label, dst)); - CHECK(elabel_indexer_.get_index(label, edge)); + label_t src = get_vertex_label_id(src_label); + label_t dst = get_vertex_label_id(dst_label); + label_t edge = get_edge_label_id(label); uint32_t index = generate_edge_label(src, dst, edge); return oe_strategy_.at(index); } @@ -273,10 +293,9 @@ EdgeStrategy Schema::get_outgoing_edge_strategy( EdgeStrategy Schema::get_incoming_edge_strategy( const std::string& src_label, const std::string& dst_label, const std::string& label) const { - label_t src, dst, edge; - CHECK(vlabel_indexer_.get_index(src_label, src)); - CHECK(vlabel_indexer_.get_index(dst_label, dst)); - CHECK(elabel_indexer_.get_index(label, edge)); + label_t src = get_vertex_label_id(src_label); + label_t dst = get_vertex_label_id(dst_label); + label_t edge = get_edge_label_id(label); uint32_t index = generate_edge_label(src, dst, edge); return ie_strategy_.at(index); } @@ -284,10 +303,9 @@ EdgeStrategy Schema::get_incoming_edge_strategy( bool Schema::outgoing_edge_mutable(const std::string& src_label, const std::string& dst_label, const std::string& label) const { - label_t src, dst, edge; - CHECK(vlabel_indexer_.get_index(src_label, src)); - CHECK(vlabel_indexer_.get_index(dst_label, dst)); - CHECK(elabel_indexer_.get_index(label, edge)); + label_t src = get_vertex_label_id(src_label); + label_t dst = get_vertex_label_id(dst_label); + label_t edge = get_edge_label_id(label); uint32_t index = generate_edge_label(src, dst, edge); return oe_mutability_.at(index); } @@ -295,10 +313,9 @@ bool Schema::outgoing_edge_mutable(const std::string& src_label, bool Schema::incoming_edge_mutable(const std::string& src_label, const std::string& dst_label, const std::string& label) const { - label_t src, dst, edge; - CHECK(vlabel_indexer_.get_index(src_label, src)); - CHECK(vlabel_indexer_.get_index(dst_label, dst)); - CHECK(elabel_indexer_.get_index(label, edge)); + label_t src = get_vertex_label_id(src_label); + label_t dst = get_vertex_label_id(dst_label); + label_t edge = get_edge_label_id(label); uint32_t index = generate_edge_label(src, dst, edge); return ie_mutability_.at(index); } @@ -306,18 +323,22 @@ bool Schema::incoming_edge_mutable(const std::string& src_label, bool Schema::get_sort_on_compaction(const std::string& src_label, const std::string& dst_label, const std::string& label) const { - label_t src, dst, edge; - CHECK(vlabel_indexer_.get_index(src_label, src)); - CHECK(vlabel_indexer_.get_index(dst_label, dst)); - CHECK(elabel_indexer_.get_index(label, edge)); + label_t src = get_vertex_label_id(src_label); + label_t dst = get_vertex_label_id(dst_label); + label_t edge = get_edge_label_id(label); uint32_t index = generate_edge_label(src, dst, edge); - CHECK(sort_on_compactions_.find(index) != sort_on_compactions_.end()); + THROW_EXCEPTION_IF( + sort_on_compactions_.find(index) == sort_on_compactions_.end(), + "Fail to get sort on compaction: " + std::to_string(index) + + ", out of range of sort_on_compactions_ " + + std::to_string(sort_on_compactions_.size())); return sort_on_compactions_.at(index); } label_t Schema::get_edge_label_id(const std::string& label) const { label_t ret; - CHECK(elabel_indexer_.get_index(label, ret)); + THROW_EXCEPTION_IF(!elabel_indexer_.get_index(label, ret), + "Edge label " + label + " not found"); return ret; } @@ -328,23 +349,26 @@ bool Schema::contains_edge_label(const std::string& label) const { std::string Schema::get_vertex_label_name(label_t index) const { std::string ret; - vlabel_indexer_.get_key(index, ret); + THROW_EXCEPTION_IF( + !vlabel_indexer_.get_key(index, ret), + "No vertex label found for label id: " + std::to_string(index)); return ret; } std::string Schema::get_edge_label_name(label_t index) const { std::string ret; - elabel_indexer_.get_key(index, ret); + THROW_EXCEPTION_IF( + !elabel_indexer_.get_key(index, ret), + "No edge label found for label id: " + std::to_string(index)); return ret; } const std::vector>& Schema::get_vertex_primary_key(label_t index) const { - if (v_primary_keys_.size() > index) { - return v_primary_keys_.at(index); - } - throw std::runtime_error("Fail to get vertex primary key: " + - std::to_string(index)); + THROW_EXCEPTION_IF(index >= v_primary_keys_.size(), + "Fail to get vertex primary key: " + + std::to_string(index) + ", out of range"); + return v_primary_keys_[index]; } // Note that plugin_dir_ and plugin_name_to_path_and_id_ are not serialized. @@ -1296,7 +1320,8 @@ std::string Schema::GetVersion() const { return version_; } bool Schema::vertex_has_property(const std::string& label, const std::string& prop) const { auto v_label_id = get_vertex_label_id(label); - CHECK(v_label_id < vprop_names_.size()); + THROW_EXCEPTION_IF(v_label_id >= vprop_names_.size(), + "vertex label id out of range of vprop_names_"); auto& v_prop_names = vprop_names_[v_label_id]; return std::find(v_prop_names.begin(), v_prop_names.end(), prop) != v_prop_names.end() || @@ -1306,7 +1331,8 @@ bool Schema::vertex_has_property(const std::string& label, bool Schema::vertex_has_primary_key(const std::string& label, const std::string& prop) const { auto v_label_id = get_vertex_label_id(label); - CHECK(v_label_id < vprop_names_.size()); + THROW_EXCEPTION_IF(v_label_id >= v_primary_keys_.size(), + "vertex label id out of range of v_primary_keys_"); auto& keys = v_primary_keys_[v_label_id]; for (size_t i = 0; i < keys.size(); ++i) { if (std::get<1>(keys[i]) == prop) { diff --git a/flex/storages/rt_mutable_graph/schema.h b/flex/storages/rt_mutable_graph/schema.h index 99ba27a1157b..f2ce4107e0d8 100644 --- a/flex/storages/rt_mutable_graph/schema.h +++ b/flex/storages/rt_mutable_graph/schema.h @@ -31,11 +31,13 @@ class Schema { // How many built-in plugins are there. // Currently only one builtin plugin, SERVER_APP is supported. static constexpr uint8_t RESERVED_PLUGIN_NUM = 1; - static constexpr uint8_t MAX_PLUGIN_ID = 253; + static constexpr uint8_t MAX_PLUGIN_ID = 252; static constexpr uint8_t HQPS_ADHOC_READ_PLUGIN_ID = 254; static constexpr uint8_t HQPS_ADHOC_WRITE_PLUGIN_ID = 255; + static constexpr uint8_t ADHOC_READ_PLUGIN_ID = 253; static constexpr const char* HQPS_ADHOC_READ_PLUGIN_ID_STR = "\xFE"; static constexpr const char* HQPS_ADHOC_WRITE_PLUGIN_ID_STR = "\xFF"; + static constexpr const char* ADHOC_READ_PLUGIN_ID_STR = "\xFD"; static constexpr const char* PRIMITIVE_TYPE_KEY = "primitive_type"; static constexpr const char* VARCHAR_KEY = "varchar"; static constexpr const char* MAX_LENGTH_KEY = "max_length"; diff --git a/flex/tests/hqps/hqps_adhoc_test.sh b/flex/tests/hqps/hqps_adhoc_test.sh index 3e37629b30a6..a5f9968fddfb 100644 --- a/flex/tests/hqps/hqps_adhoc_test.sh +++ b/flex/tests/hqps/hqps_adhoc_test.sh @@ -104,7 +104,11 @@ start_engine_service(){ fi cmd="${SERVER_BIN} -c ${ENGINE_CONFIG_PATH} -g ${GRAPH_SCHEMA_YAML} " cmd="${cmd} --data-path ${GRAPH_CSR_DATA_DIR} " - + + if [ "${TEST_TYPE}" == "gremlin" ]; then + cmd="${cmd} --enable-adhoc-handler=true" + fi + info "Start engine service with command: ${cmd}" ${cmd} & sleep 5 diff --git a/flex/utils/app_utils.cc b/flex/utils/app_utils.cc index d138080ef5ea..6490c195265c 100644 --- a/flex/utils/app_utils.cc +++ b/flex/utils/app_utils.cc @@ -53,6 +53,10 @@ void Encoder::put_int_at(size_t pos, int v) { void Encoder::put_byte(uint8_t v) { buf_.push_back(static_cast(v)); } +void Encoder::put_bytes(const char* data, size_t size) { + buf_.insert(buf_.end(), data, data + size); +} + size_t Encoder::skip_byte() { size_t size = buf_.size(); buf_.resize(size + 1); @@ -135,7 +139,11 @@ double Decoder::get_double() { data_ += 8; return ret; } - +std::string_view Decoder::get_bytes() { + std::string_view ret(data_, end_ - data_); + data_ = end_; + return ret; +} std::string_view Decoder::get_string() { int len = get_int(); std::string_view ret(data_, len); diff --git a/flex/utils/app_utils.h b/flex/utils/app_utils.h index a8156376dc95..b5e6d71f0222 100644 --- a/flex/utils/app_utils.h +++ b/flex/utils/app_utils.h @@ -40,6 +40,8 @@ class Encoder { void put_byte(uint8_t v); + void put_bytes(const char* data, size_t size); + size_t skip_byte(); void put_byte_at(size_t pos, uint8_t v); @@ -71,6 +73,8 @@ class Decoder { std::string_view get_string(); + std::string_view get_bytes(); + std::string_view get_small_string(); uint8_t get_byte(); diff --git a/flex/utils/property/column.cc b/flex/utils/property/column.cc index cf11e23f1bc3..85a1878c9a6b 100644 --- a/flex/utils/property/column.cc +++ b/flex/utils/property/column.cc @@ -144,7 +144,9 @@ std::shared_ptr CreateColumn(PropertyType type, return nullptr; } } else { - if (type == PropertyType::kBool) { + if (type == PropertyType::kEmpty) { + return std::make_shared>(strategy); + } else if (type == PropertyType::kBool) { return std::make_shared(strategy); } else if (type == PropertyType::kInt32) { return std::make_shared(strategy); diff --git a/flex/utils/property/column.h b/flex/utils/property/column.h index 5d57d524c1ea..f694a1b05406 100644 --- a/flex/utils/property/column.h +++ b/flex/utils/property/column.h @@ -237,6 +237,37 @@ using DayColumn = TypedColumn; using DoubleColumn = TypedColumn; using FloatColumn = TypedColumn; +template <> +class TypedColumn : public ColumnBase { + public: + TypedColumn(StorageStrategy strategy) : strategy_(strategy) {} + ~TypedColumn() {} + + void open(const std::string& name, const std::string& snapshot_dir, + const std::string& work_dir) override {} + void open_in_memory(const std::string& name) override {} + void open_with_hugepages(const std::string& name, bool force) override {} + void touch(const std::string& filename) override {} + void dump(const std::string& filename) override {} + void copy_to_tmp(const std::string& cur_path, + const std::string& tmp_path) override {} + void close() override {} + size_t size() const override { return 0; } + void resize(size_t size) override {} + + PropertyType type() const override { return PropertyType::kEmpty; } + + void set_any(size_t index, const Any& value) override {} + + Any get(size_t index) const override { return Any(); } + + void ingest(uint32_t index, grape::OutArchive& arc) override {} + + StorageStrategy storage_strategy() const override { return strategy_; } + + private: + StorageStrategy strategy_; +}; template <> class TypedColumn : public ColumnBase { public: diff --git a/interactive_engine/assembly/src/bin/groot/store_ctl.sh b/interactive_engine/assembly/src/bin/groot/store_ctl.sh index 655b00875dd5..2342adff0693 100755 --- a/interactive_engine/assembly/src/bin/groot/store_ctl.sh +++ b/interactive_engine/assembly/src/bin/groot/store_ctl.sh @@ -27,21 +27,13 @@ _setup_env() { GROOT_HOME=$(dirname "$SCRIPT_DIR") fi - if [ -z "${GROOT_LOGBACK_FILE}" ]; then - readonly GROOT_LOGBACK_FILE="${GROOT_HOME}/conf/logback.xml" - fi - - if [ -z "${GROOT_CONF_FILE}" ]; then - readonly GROOT_CONF_FILE="${GROOT_HOME}/conf/groot.config" - fi - - if [ -z "${LOG_NAME}" ]; then - readonly LOG_NAME="graphscope-store" - fi - - if [ -z "${LOG_DIR}" ]; then - export LOG_DIR="/var/log/graphscope" - fi + export GROOT_LOGBACK_FILE=${GROOT_LOGBACK_FILE:-${GROOT_HOME}/conf/logback.xml} + export GROOT_CONF_FILE=${GROOT_CONF_FILE:-${GROOT_HOME}/conf/groot.config} + export LOG_NAME=${LOG_NAME:-graphscope-store} + export LOG_DIR=${LOG_DIR:-/var/log/graphscope} + export LOG_MAX_FILE_SIZE=${LOG_MAX_FILE_SIZE:-100MB} + export LOG_MAX_HISTORY=${LOG_MAX_HISTORY:-10} + export LOG_TOTAL_SIZE_CAP=${LOG_TOTAL_SIZE_CAP:-1GB} mkdir -p ${LOG_DIR} diff --git a/interactive_engine/assembly/src/conf/graphscope/log4j2.xml b/interactive_engine/assembly/src/conf/graphscope/log4j2.xml index 0a9dad7a26f1..17b810929a2d 100644 --- a/interactive_engine/assembly/src/conf/graphscope/log4j2.xml +++ b/interactive_engine/assembly/src/conf/graphscope/log4j2.xml @@ -3,11 +3,11 @@ - + - + @@ -24,10 +24,8 @@ - - diff --git a/interactive_engine/assembly/src/conf/graphscope/log4rs.yml b/interactive_engine/assembly/src/conf/graphscope/log4rs.yml index 320c8a37a90b..93319ac1a95a 100644 --- a/interactive_engine/assembly/src/conf/graphscope/log4rs.yml +++ b/interactive_engine/assembly/src/conf/graphscope/log4rs.yml @@ -4,7 +4,7 @@ appenders: stdout: kind: console encoder: - pattern: "{d(%Y-%m-%d %H:%M:%S.%f)} {h({l:<5})} (({f}:{L})) [{T}] {m}{n}" + pattern: "{d(%Y-%m-%d %H:%M:%S.%f)} {h({l:<5})} (({f}:{L})) [{T}][{X(trace)}] {m}{n}" root: level: info diff --git a/interactive_engine/assembly/src/conf/groot/log4rs.yml b/interactive_engine/assembly/src/conf/groot/log4rs.yml index b14c5bc40d7c..0278475a7499 100644 --- a/interactive_engine/assembly/src/conf/groot/log4rs.yml +++ b/interactive_engine/assembly/src/conf/groot/log4rs.yml @@ -1,22 +1,22 @@ -refresh_rate: 30 seconds +refresh_rate: 60 seconds appenders: stdout: kind: console encoder: - pattern: "{d(%Y-%m-%d %H:%M:%S.%f)} {h({l:<5})} (({f}:{L})) [{T}] {m}{n}" + pattern: "{d(%Y-%m-%d %H:%M:%S.%f)} {h({l:<5})} (({f}:{L})) [{T}] {m}{n} {X(trace)}" file: kind: rolling_file append: true path: "$ENV{LOG_DIR}/store-executor.log" encoder: - pattern: "{d(%Y-%m-%d %H:%M:%S.%f)} {h({l:<5})} (({f}:{L})) [{T}] {m}{n}" + pattern: "{d(%Y-%m-%d %H:%M:%S.%f)} {h({l:<5})} (({f}:{L})) [{T}] {m}{n} {X(trace)}" policy: trigger: kind: size limit: 100mb roller: kind: fixed_window - pattern: "$ENV{LOG_DIR}/store-executor.log.{}" + pattern: "$ENV{LOG_DIR}/store-executor.{}.log" count: 10 root: diff --git a/interactive_engine/assembly/src/conf/groot/logback.xml b/interactive_engine/assembly/src/conf/groot/logback.xml index 251f6a8990b9..05953484c7ef 100644 --- a/interactive_engine/assembly/src/conf/groot/logback.xml +++ b/interactive_engine/assembly/src/conf/groot/logback.xml @@ -1,29 +1,31 @@ - - - + + + + + + ${log_dir}/${log_name}.log - ${log_dir}/${log_name}.%d{yyyy-MM-dd}.%i.log - 100MB - 10 - 1GB + ${log_dir}/${log_name}.%d{yyyy-MM-dd}.%i.gz + ${log_max_file_size} + ${log_max_history} + ${log_total_size_cap} - [%d{ISO8601}][%p][%t][%c:%L] %m%n + [%d{ISO8601}][%p][%t][%c{3}] %m%n %X - [%d{ISO8601}][%p][%t][%c:%L] %m%n + [%d{ISO8601}][%p][%t][%c{3}] %m%n %X ${log_dir}/metric.log - - ${log_dir}/metric.log.%d{yyyy-MM-dd}.%i.log - 100MB - 5 + + ${log_dir}/metric.log.%d{yyyy-MM-dd}.%i.gz + 7 500MB diff --git a/interactive_engine/benchmark/queries/cypher_queries/Qc1(a).cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qc1(a).cypher similarity index 100% rename from interactive_engine/benchmark/queries/cypher_queries/Qc1(a).cypher rename to interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qc1(a).cypher diff --git a/interactive_engine/benchmark/queries/cypher_queries/Qc1(b).cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qc1(b).cypher similarity index 100% rename from interactive_engine/benchmark/queries/cypher_queries/Qc1(b).cypher rename to interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qc1(b).cypher diff --git a/interactive_engine/benchmark/queries/cypher_queries/Qc2(a).cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qc2(a).cypher similarity index 100% rename from interactive_engine/benchmark/queries/cypher_queries/Qc2(a).cypher rename to interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qc2(a).cypher diff --git a/interactive_engine/benchmark/queries/cypher_queries/Qc2(b).cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qc2(b).cypher similarity index 100% rename from interactive_engine/benchmark/queries/cypher_queries/Qc2(b).cypher rename to interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qc2(b).cypher diff --git a/interactive_engine/benchmark/queries/cypher_queries/Qc3(a).cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qc3(a).cypher similarity index 100% rename from interactive_engine/benchmark/queries/cypher_queries/Qc3(a).cypher rename to interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qc3(a).cypher diff --git a/interactive_engine/benchmark/queries/cypher_queries/Qc3(b).cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qc3(b).cypher similarity index 100% rename from interactive_engine/benchmark/queries/cypher_queries/Qc3(b).cypher rename to interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qc3(b).cypher diff --git a/interactive_engine/benchmark/queries/cypher_queries/Qc4(a).cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qc4(a).cypher similarity index 100% rename from interactive_engine/benchmark/queries/cypher_queries/Qc4(a).cypher rename to interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qc4(a).cypher diff --git a/interactive_engine/benchmark/queries/cypher_queries/Qc4(b).cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qc4(b).cypher similarity index 100% rename from interactive_engine/benchmark/queries/cypher_queries/Qc4(b).cypher rename to interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qc4(b).cypher diff --git a/interactive_engine/benchmark/queries/cypher_queries/Qr1.cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qr1.cypher similarity index 100% rename from interactive_engine/benchmark/queries/cypher_queries/Qr1.cypher rename to interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qr1.cypher diff --git a/interactive_engine/benchmark/queries/cypher_queries/Qr2.cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qr2.cypher similarity index 100% rename from interactive_engine/benchmark/queries/cypher_queries/Qr2.cypher rename to interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qr2.cypher diff --git a/interactive_engine/benchmark/queries/cypher_queries/Qr3.cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qr3.cypher similarity index 100% rename from interactive_engine/benchmark/queries/cypher_queries/Qr3.cypher rename to interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qr3.cypher diff --git a/interactive_engine/benchmark/queries/cypher_queries/Qr4.cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qr4.cypher similarity index 100% rename from interactive_engine/benchmark/queries/cypher_queries/Qr4.cypher rename to interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qr4.cypher diff --git a/interactive_engine/benchmark/queries/cypher_queries/Qr5.cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qr5.cypher similarity index 100% rename from interactive_engine/benchmark/queries/cypher_queries/Qr5.cypher rename to interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qr5.cypher diff --git a/interactive_engine/benchmark/queries/cypher_queries/Qr6.cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qr6.cypher similarity index 100% rename from interactive_engine/benchmark/queries/cypher_queries/Qr6.cypher rename to interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qr6.cypher diff --git a/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qr7.cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qr7.cypher new file mode 100644 index 000000000000..4c6be7d078c6 --- /dev/null +++ b/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qr7.cypher @@ -0,0 +1,3 @@ +Match (p1: PERSON {id:1243})-[:KNOWS*3..4]->(:PERSON) +UNION (p1: PERSON {id:1243})-[:KNOWS*4..5]->(:PERSON), +Return count(p1); \ No newline at end of file diff --git a/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qr8.cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qr8.cypher new file mode 100644 index 000000000000..de3bb0b2313e --- /dev/null +++ b/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qr8.cypher @@ -0,0 +1,3 @@ +Match (p1: PERSON {id:1243})-[:KNOWS*4..5]->(:PERSON) +UNION (p1: PERSON {id:1243})-[:KNOWS*5..6]->(:PERSON), +Return count(p1); \ No newline at end of file diff --git a/interactive_engine/benchmark/queries/cypher_queries/Qt1.cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qt1.cypher similarity index 100% rename from interactive_engine/benchmark/queries/cypher_queries/Qt1.cypher rename to interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qt1.cypher diff --git a/interactive_engine/benchmark/queries/cypher_queries/Qt2.cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qt2.cypher similarity index 100% rename from interactive_engine/benchmark/queries/cypher_queries/Qt2.cypher rename to interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qt2.cypher diff --git a/interactive_engine/benchmark/queries/cypher_queries/Qt3.cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qt3.cypher similarity index 100% rename from interactive_engine/benchmark/queries/cypher_queries/Qt3.cypher rename to interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qt3.cypher diff --git a/interactive_engine/benchmark/queries/cypher_queries/Qt4.cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qt4.cypher similarity index 100% rename from interactive_engine/benchmark/queries/cypher_queries/Qt4.cypher rename to interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qt4.cypher diff --git a/interactive_engine/benchmark/queries/cypher_queries/Qt5.cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qt5.cypher similarity index 100% rename from interactive_engine/benchmark/queries/cypher_queries/Qt5.cypher rename to interactive_engine/benchmark/queries/cypher_queries/experiments/gopt/Qt5.cypher diff --git a/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc1(a).cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc1(a).cypher new file mode 100644 index 000000000000..f0602e726e02 --- /dev/null +++ b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc1(a).cypher @@ -0,0 +1,4 @@ +Match (message:Post)-[:HAS_CREATOR]->(person), + (message:Post)-[:HAS_TAG]->(tag:Tag), + (person)-[:HAS_INTEREST]->(tag:Tag) +Return count(person); diff --git a/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc1(b).cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc1(b).cypher new file mode 100644 index 000000000000..a0330fee7c19 --- /dev/null +++ b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc1(b).cypher @@ -0,0 +1,4 @@ +Match (message)-[:KNOWS|HAS_MODERATOR]->(person:Person), + (message)-[]->(tag:Tag), + (person)-[]->(tag) +Return count(person); diff --git a/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc2(a).cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc2(a).cypher new file mode 100644 index 000000000000..7095af2b83ea --- /dev/null +++ b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc2(a).cypher @@ -0,0 +1,5 @@ +Match (person1:Person)-[:LIKES]->(message:Post), + (message:Post)<-[:CONTAINER_OF]-(person2:Forum), + (person1:Person)-[:KNOWS]->(place), + (person2:Forum)-[:HAS_MODERATOR]->(place) +Return count(person1); \ No newline at end of file diff --git a/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc2(b).cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc2(b).cypher new file mode 100644 index 000000000000..40900bd64928 --- /dev/null +++ b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc2(b).cypher @@ -0,0 +1,5 @@ +Match (person1:Person)-[:LIKES]->(message:Post), + (message:Post)<-[:CONTAINER_OF]-(person2:Forum), + (person1:Person)-[:KNOWS|HAS_INTEREST]->(place), + (person2:Forum)-[:HAS_MODERATOR|HAS_TAG]->(place) +Return count(person1); \ No newline at end of file diff --git a/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc3(a).cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc3(a).cypher new file mode 100644 index 000000000000..8744f49df441 --- /dev/null +++ b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc3(a).cypher @@ -0,0 +1,5 @@ +Match (person1)<-[:HAS_CREATOR]-(comment:Comment), + (comment:Comment)-[:REPLY_OF]->(post:Post), + (post:Post)<-[:CONTAINER_OF]-(forum), + (forum)-[:HAS_MEMBER]->(person2) +Return count(person1); \ No newline at end of file diff --git a/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc3(b).cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc3(b).cypher new file mode 100644 index 000000000000..50b5115a0f1a --- /dev/null +++ b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc3(b).cypher @@ -0,0 +1,4 @@ +Match (p:Comment)-[]->(:Person)-[]->(:Place), + (p)<-[]-(message), + (message)-[]->(tag:Tag) +Return count(p); \ No newline at end of file diff --git a/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc4(a).cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc4(a).cypher new file mode 100644 index 000000000000..0efd33b6581a --- /dev/null +++ b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc4(a).cypher @@ -0,0 +1,7 @@ +Match (forum)-[:CONTAINER_OF]->(post:Post), + (forum)-[:HAS_MEMBER]->(person1:Person), + (forum)-[:HAS_MEMBER]->(person2:Person), + (person1:Person)-[:KNOWS]->(person2:Person), + (person1:Person)-[:LIKES]->(post:Post), + (person2:Person)-[:LIKES]->(post:Post) +Return count(person1); \ No newline at end of file diff --git a/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc4(b).cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc4(b).cypher new file mode 100644 index 000000000000..4f18cdb43be9 --- /dev/null +++ b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qc4(b).cypher @@ -0,0 +1,7 @@ +Match (forum)-[:HAS_TAG]->(post:Tag), + (forum)-[:HAS_MODERATOR]->(person1), + (forum)-[:HAS_MODERATOR|CONTAINER_OF]->(person2), + (person1)-[:KNOWS|LIKES]->(person2), + (person1)-[:HAS_INTEREST]->(post:Tag), + (person2)-[:HAS_INTEREST|HAS_TAG]->(post:Tag) +Return count(person1); \ No newline at end of file diff --git a/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qr1.cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qr1.cypher new file mode 100644 index 000000000000..63eb639aae0a --- /dev/null +++ b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qr1.cypher @@ -0,0 +1,3 @@ +Match (p1:Person)-[:KNOWS]->(p2:Person) +Where p1.id < 933 +Return count(p1); \ No newline at end of file diff --git a/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qr2.cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qr2.cypher new file mode 100644 index 000000000000..1b9e67c2329e --- /dev/null +++ b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qr2.cypher @@ -0,0 +1,5 @@ +Match (p1:Person)<-[:HAS_MODERATOR]-(forum:Forum), + (p1:Person)<-[:HAS_CREATOR]-(post:Post), + (post)<-[:CONTAINER_OF]-(forum) +Where p1.id < 933 +Return count(p1); \ No newline at end of file diff --git a/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qr3.cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qr3.cypher new file mode 100644 index 000000000000..33a6aeabcaa1 --- /dev/null +++ b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qr3.cypher @@ -0,0 +1 @@ +Match (p1:Person)-[:KNOWS]->(p2:Person)-[:KNOWS]->(p3:Person) Return count(p1); \ No newline at end of file diff --git a/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qr4.cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qr4.cypher new file mode 100644 index 000000000000..e669e04dc6a0 --- /dev/null +++ b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qr4.cypher @@ -0,0 +1 @@ +Match (p1:Person)<-[:HAS_MODERATOR]-(forum:Forum)-[:CONTAINER_OF]->(post:Post)-[:HAS_CREATOR]->(p2:Person) Return count(p1); \ No newline at end of file diff --git a/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qt1.cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qt1.cypher new file mode 100644 index 000000000000..a685b75f3ded --- /dev/null +++ b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qt1.cypher @@ -0,0 +1 @@ +Match (p1:Person)<-[:HAS_CREATOR]-()<-[:CONTAINER_OF]-(p2) Return count(p1); \ No newline at end of file diff --git a/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qt2.cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qt2.cypher new file mode 100644 index 000000000000..4bacf99d040b --- /dev/null +++ b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qt2.cypher @@ -0,0 +1 @@ +Match (p)-[]->(:Organisation)-[]->(:Place) Return count(p); \ No newline at end of file diff --git a/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qt3.cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qt3.cypher new file mode 100644 index 000000000000..d12671012651 --- /dev/null +++ b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qt3.cypher @@ -0,0 +1 @@ +Match (p1)<-[:IS_LOCATED_IN]-(p2:Comment)-[]->(:Tag) Return count(p1); \ No newline at end of file diff --git a/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qt4.cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qt4.cypher new file mode 100644 index 000000000000..223fddf6f96d --- /dev/null +++ b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qt4.cypher @@ -0,0 +1,3 @@ +Match (p1)<-[]-(p2:Post), + (p1)<-[:HAS_MODERATOR]-()-[]->(p2) +Return count(p1); diff --git a/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qt5.cypher b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qt5.cypher new file mode 100644 index 000000000000..2543e68b708b --- /dev/null +++ b/interactive_engine/benchmark/queries/cypher_queries/experiments/neo4j/Qt5.cypher @@ -0,0 +1 @@ +Match (p:Post)-[]->(p1), (p1)-[]->(:Place) Return count(p1); \ No newline at end of file diff --git a/interactive_engine/common/pom.xml b/interactive_engine/common/pom.xml index 38c3464d449a..2590c6fe6d70 100644 --- a/interactive_engine/common/pom.xml +++ b/interactive_engine/common/pom.xml @@ -121,6 +121,7 @@ ddl_service.proto write_service.proto request_option.proto + error/insight.proto diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/RoleType.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/RoleType.java index 1125ab52e9a5..d1a919a0639d 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/RoleType.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/RoleType.java @@ -13,6 +13,8 @@ */ package com.alibaba.graphscope.groot.common; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; + import java.util.HashMap; import java.util.Map; @@ -46,7 +48,7 @@ public String getName() { public static RoleType fromName(String roleName) { RoleType roleType = lookup.get(roleName); if (roleType == null || roleType == RoleType.UNKNOWN) { - throw new IllegalArgumentException("Unknown RoleType: [" + roleName + "]"); + throw new InvalidArgumentException("Unknown RoleType: [" + roleName + "]"); } return roleType; } diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/config/Config.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/config/Config.java index f18c444de55d..f636e377b68e 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/config/Config.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/config/Config.java @@ -15,6 +15,8 @@ */ package com.alibaba.graphscope.groot.common.config; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; + import java.util.function.Function; public class Config { @@ -35,7 +37,7 @@ public T get(Configs configs) { T val = parseFunc.apply(valStr); return val; } catch (Exception e) { - throw new IllegalArgumentException( + throw new InvalidArgumentException( "key [" + key + "] val [" + valStr + "] parse failed", e); } } diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/AlreadyExistsException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/AlreadyExistsException.java new file mode 100644 index 000000000000..1f0ed751da90 --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/AlreadyExistsException.java @@ -0,0 +1,21 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class AlreadyExistsException extends GrootException { + public AlreadyExistsException(Throwable t) { + super(Code.ALREADY_EXISTS, t); + } + + public AlreadyExistsException(String msg) { + super(Code.ALREADY_EXISTS, msg); + } + + public AlreadyExistsException(String msg, Throwable t) { + super(Code.ALREADY_EXISTS, msg, t); + } + + public AlreadyExistsException() { + super(Code.ALREADY_EXISTS); + } +} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/BackupException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/BackupException.java index 58b0ed6c514a..b3ee2ef18de8 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/BackupException.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/BackupException.java @@ -15,8 +15,22 @@ */ package com.alibaba.graphscope.groot.common.exception; +import com.alibaba.graphscope.proto.Code; + public class BackupException extends GrootException { public BackupException(String msg) { - super(msg); + super(Code.BACKUP_FAILED, msg); + } + + public BackupException(String msg, Throwable t) { + super(Code.BACKUP_FAILED, msg, t); + } + + public BackupException() { + super(Code.BACKUP_FAILED); + } + + public BackupException(Throwable t) { + super(Code.BACKUP_FAILED, t); } } diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/CancelledException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/CancelledException.java new file mode 100644 index 000000000000..0209db3abc6e --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/CancelledException.java @@ -0,0 +1,21 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class CancelledException extends GrootException { + public CancelledException(String msg) { + super(Code.CANCELLED, msg); + } + + public CancelledException(String msg, Throwable t) { + super(Code.CANCELLED, msg, t); + } + + public CancelledException() { + super(Code.CANCELLED); + } + + public CancelledException(Throwable t) { + super(Code.CANCELLED, t); + } +} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/CompilationFailureException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/CompilationFailureException.java new file mode 100644 index 000000000000..0d267ba2df3a --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/CompilationFailureException.java @@ -0,0 +1,21 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class CompilationFailureException extends GrootException { + public CompilationFailureException(Throwable t) { + super(Code.COMPILATION_FAILURE, t); + } + + public CompilationFailureException(String msg) { + super(Code.COMPILATION_FAILURE, msg); + } + + public CompilationFailureException(String msg, Throwable t) { + super(Code.COMPILATION_FAILURE, msg, t); + } + + public CompilationFailureException() { + super(Code.COMPILATION_FAILURE); + } +} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/DataNotExistsException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/DataNotExistsException.java new file mode 100644 index 000000000000..48fccea7ddb3 --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/DataNotExistsException.java @@ -0,0 +1,21 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class DataNotExistsException extends GrootException { + public DataNotExistsException(Throwable t) { + super(Code.DATA_NOT_EXISTS, t); + } + + public DataNotExistsException(String msg) { + super(Code.DATA_NOT_EXISTS, msg); + } + + public DataNotExistsException(String msg, Throwable t) { + super(Code.DATA_NOT_EXISTS, msg, t); + } + + public DataNotExistsException() { + super(Code.DATA_NOT_EXISTS); + } +} diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/request/DdlException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/DdlException.java similarity index 69% rename from interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/request/DdlException.java rename to interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/DdlException.java index 2344d954f501..b3d234dad869 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/request/DdlException.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/DdlException.java @@ -13,17 +13,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package com.alibaba.graphscope.groot.schema.request; +package com.alibaba.graphscope.groot.common.exception; -import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.proto.Code; public class DdlException extends GrootException { + public DdlException(Throwable t) { + super(Code.DDL_ERROR, t); + } + public DdlException(String msg) { - super(msg); + super(Code.DDL_ERROR, msg); } - public DdlException(Throwable t) { - super(t); + public DdlException(String msg, Throwable t) { + super(Code.DDL_ERROR, msg, t); + } + + public DdlException() { + super(Code.DDL_ERROR); } } diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/ExternalStorageErrorException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/ExternalStorageErrorException.java new file mode 100644 index 000000000000..d4d3d141b314 --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/ExternalStorageErrorException.java @@ -0,0 +1,21 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class ExternalStorageErrorException extends GrootException { + public ExternalStorageErrorException(Throwable t) { + super(Code.EXTERNAL_STORAGE_ERROR, t); + } + + public ExternalStorageErrorException(String msg) { + super(Code.EXTERNAL_STORAGE_ERROR, msg); + } + + public ExternalStorageErrorException(String msg, Throwable t) { + super(Code.EXTERNAL_STORAGE_ERROR, msg, t); + } + + public ExternalStorageErrorException() { + super(Code.EXTERNAL_STORAGE_ERROR); + } +} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/GaiaInternalException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/GaiaInternalException.java new file mode 100644 index 000000000000..cf16897840c8 --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/GaiaInternalException.java @@ -0,0 +1,21 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class GaiaInternalException extends GrootException { + public GaiaInternalException(Throwable t) { + super(Code.GAIA_INTERNAL_ERROR, t); + } + + public GaiaInternalException(String msg) { + super(Code.GAIA_INTERNAL_ERROR, msg); + } + + public GaiaInternalException(String msg, Throwable t) { + super(Code.GAIA_INTERNAL_ERROR, msg, t); + } + + public GaiaInternalException() { + super(Code.GAIA_INTERNAL_ERROR); + } +} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/GrootException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/GrootException.java index e94fdb40362f..ae82c659ebe2 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/GrootException.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/GrootException.java @@ -15,20 +15,31 @@ */ package com.alibaba.graphscope.groot.common.exception; -import com.alibaba.graphscope.groot.common.util.ExceptionUtils; +import com.alibaba.graphscope.proto.Code; public class GrootException extends RuntimeException { + private final int componentCode = 6; private int errorCode; - public GrootException(ExceptionUtils.ErrorCode code, String msg) { + public GrootException(Code code) { + super(); + this.errorCode = code.getNumber(); + } + + public GrootException(Code code, Throwable t) { + super(t); + this.errorCode = code.getNumber(); + } + + public GrootException(Code code, String msg) { super(msg); - this.errorCode = code.toInt(); + this.errorCode = code.getNumber(); } - public GrootException(ExceptionUtils.ErrorCode code, Throwable e, String msg) { - super(msg, e); - this.errorCode = code.toInt(); + public GrootException(Code code, String msg, Throwable t) { + super(msg, t); + this.errorCode = code.getNumber(); } public int getErrorCode() { @@ -54,4 +65,10 @@ public GrootException(String msg, Throwable t) { public GrootException() { super(); } + + @Override + public String getMessage() { + String ec = String.format("%02d-%04d", componentCode, errorCode); + return String.format("ec=%s, %s", ec, super.getMessage()); + } } diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/GraphPropertyNotFoundException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/IllegalSchemaException.java similarity index 60% rename from interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/GraphPropertyNotFoundException.java rename to interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/IllegalSchemaException.java index 88bc566b45c2..d650e3921794 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/GraphPropertyNotFoundException.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/IllegalSchemaException.java @@ -15,20 +15,23 @@ */ package com.alibaba.graphscope.groot.common.exception; -public class GraphPropertyNotFoundException extends GraphSchemaException { - public GraphPropertyNotFoundException() { - super(); +import com.alibaba.graphscope.proto.Code; + +public class IllegalSchemaException extends GrootException { + + public IllegalSchemaException(String msg) { + super(Code.ILLEGAL_SCHEMA, msg); } - public GraphPropertyNotFoundException(Exception exception) { - super(exception); + public IllegalSchemaException(String msg, Throwable t) { + super(Code.ILLEGAL_SCHEMA, msg, t); } - public GraphPropertyNotFoundException(String message) { - super(message); + public IllegalSchemaException() { + super(Code.ILLEGAL_SCHEMA); } - public GraphPropertyNotFoundException(String message, Exception exception) { - super(message, exception); + public IllegalSchemaException(Throwable t) { + super(Code.ILLEGAL_SCHEMA, t); } } diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/IllegalStateException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/IllegalStateException.java new file mode 100644 index 000000000000..8ee044072dfc --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/IllegalStateException.java @@ -0,0 +1,21 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class IllegalStateException extends GrootException { + public IllegalStateException(Throwable t) { + super(Code.ILLEGAL_STATE, t); + } + + public IllegalStateException(String msg) { + super(Code.ILLEGAL_STATE, msg); + } + + public IllegalStateException(String msg, Throwable t) { + super(Code.ILLEGAL_STATE, msg, t); + } + + public IllegalStateException() { + super(Code.ILLEGAL_STATE); + } +} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/IngestRejectException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/IngestRejectException.java deleted file mode 100644 index bb0be06347c3..000000000000 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/IngestRejectException.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.graphscope.groot.common.exception; - -public class IngestRejectException extends GrootException { - - public IngestRejectException(String msg) { - super(msg); - } -} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/InternalException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/InternalException.java new file mode 100644 index 000000000000..c7d916411165 --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/InternalException.java @@ -0,0 +1,21 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class InternalException extends GrootException { + public InternalException(Throwable t) { + super(Code.INTERNAL, t); + } + + public InternalException(String msg) { + super(Code.INTERNAL, msg); + } + + public InternalException(String msg, Throwable t) { + super(Code.INTERNAL, msg, t); + } + + public InternalException() { + super(Code.INTERNAL); + } +} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/InvalidArgumentException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/InvalidArgumentException.java new file mode 100644 index 000000000000..d3fc44281b36 --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/InvalidArgumentException.java @@ -0,0 +1,22 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class InvalidArgumentException extends GrootException { + + public InvalidArgumentException(Throwable t) { + super(Code.INVALID_ARGUMENT, t); + } + + public InvalidArgumentException(String msg) { + super(Code.INVALID_ARGUMENT, msg); + } + + public InvalidArgumentException(String msg, Throwable t) { + super(Code.INVALID_ARGUMENT, msg, t); + } + + public InvalidArgumentException() { + super(Code.INVALID_ARGUMENT); + } +} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/InvalidDataException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/InvalidDataException.java index 6a6746ca580c..2aeeb444891f 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/InvalidDataException.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/InvalidDataException.java @@ -15,9 +15,23 @@ */ package com.alibaba.graphscope.groot.common.exception; +import com.alibaba.graphscope.proto.Code; + public class InvalidDataException extends GrootException { public InvalidDataException(String msg) { - super(msg); + super(Code.INVALID_DATA, msg); + } + + public InvalidDataException(String msg, Throwable t) { + super(Code.INVALID_DATA, msg, t); + } + + public InvalidDataException() { + super(Code.INVALID_DATA); + } + + public InvalidDataException(Throwable t) { + super(Code.INVALID_DATA, t); } } diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/InvalidDataTypeException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/InvalidDataTypeException.java new file mode 100644 index 000000000000..b5f32558f218 --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/InvalidDataTypeException.java @@ -0,0 +1,21 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class InvalidDataTypeException extends GrootException { + public InvalidDataTypeException(Throwable t) { + super(Code.INVALID_DATA_TYPE, t); + } + + public InvalidDataTypeException(String msg) { + super(Code.INVALID_DATA_TYPE, msg); + } + + public InvalidDataTypeException(String msg, Throwable t) { + super(Code.INVALID_DATA_TYPE, msg, t); + } + + public InvalidDataTypeException() { + super(Code.INVALID_DATA_TYPE); + } +} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/InvalidOperationException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/InvalidOperationException.java new file mode 100644 index 000000000000..df06fc0cebc6 --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/InvalidOperationException.java @@ -0,0 +1,21 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class InvalidOperationException extends GrootException { + public InvalidOperationException(Throwable t) { + super(Code.INVALID_OPERATION, t); + } + + public InvalidOperationException(String msg) { + super(Code.INVALID_OPERATION, msg); + } + + public InvalidOperationException(String msg, Throwable t) { + super(Code.INVALID_OPERATION, msg, t); + } + + public InvalidOperationException() { + super(Code.INVALID_OPERATION); + } +} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/LockFailedException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/LockFailedException.java new file mode 100644 index 000000000000..a39c56ce7ecb --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/LockFailedException.java @@ -0,0 +1,22 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class LockFailedException extends GrootException { + + public LockFailedException(Throwable t) { + super(Code.LOCK_FAILED, t); + } + + public LockFailedException(String msg) { + super(Code.LOCK_FAILED, msg); + } + + public LockFailedException(String msg, Throwable t) { + super(Code.LOCK_FAILED, msg, t); + } + + public LockFailedException() { + super(Code.LOCK_FAILED); + } +} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/NetworkFailureException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/NetworkFailureException.java new file mode 100644 index 000000000000..b144bbb07a24 --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/NetworkFailureException.java @@ -0,0 +1,21 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class NetworkFailureException extends GrootException { + public NetworkFailureException(Throwable t) { + super(Code.NETWORK_FAILURE, t); + } + + public NetworkFailureException(String msg) { + super(Code.NETWORK_FAILURE, msg); + } + + public NetworkFailureException(String msg, Throwable t) { + super(Code.NETWORK_FAILURE, msg, t); + } + + public NetworkFailureException() { + super(Code.NETWORK_FAILURE); + } +} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/NodeConnectException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/NodeConnectException.java deleted file mode 100644 index 29b0f598400a..000000000000 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/NodeConnectException.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.graphscope.groot.common.exception; - -public class NodeConnectException extends GrootException { - public NodeConnectException(String msg) { - super(msg); - } -} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/NotFoundException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/NotFoundException.java new file mode 100644 index 000000000000..002eb2fe0720 --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/NotFoundException.java @@ -0,0 +1,21 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class NotFoundException extends GrootException { + public NotFoundException(Throwable t) { + super(Code.NOT_FOUND, t); + } + + public NotFoundException(String msg) { + super(Code.NOT_FOUND, msg); + } + + public NotFoundException(String msg, Throwable t) { + super(Code.NOT_FOUND, msg, t); + } + + public NotFoundException() { + super(Code.NOT_FOUND); + } +} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/PermissionDeniedException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/PermissionDeniedException.java new file mode 100644 index 000000000000..d806aaaa3ec9 --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/PermissionDeniedException.java @@ -0,0 +1,21 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class PermissionDeniedException extends GrootException { + public PermissionDeniedException(Throwable t) { + super(Code.PERMISSION_DENIED, t); + } + + public PermissionDeniedException(String msg) { + super(Code.PERMISSION_DENIED, msg); + } + + public PermissionDeniedException(String msg, Throwable t) { + super(Code.PERMISSION_DENIED, msg, t); + } + + public PermissionDeniedException() { + super(Code.PERMISSION_DENIED); + } +} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/PortInUseException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/PortInUseException.java new file mode 100644 index 000000000000..06c102aa1f18 --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/PortInUseException.java @@ -0,0 +1,21 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class PortInUseException extends GrootException { + public PortInUseException(Throwable t) { + super(Code.PORT_IN_USE, t); + } + + public PortInUseException(String msg) { + super(Code.PORT_IN_USE, msg); + } + + public PortInUseException(String msg, Throwable t) { + super(Code.PORT_IN_USE, msg, t); + } + + public PortInUseException() { + super(Code.PORT_IN_USE); + } +} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/PropertyDefNotFoundException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/PropertyDefNotFoundException.java deleted file mode 100644 index b64f36a2fc2f..000000000000 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/PropertyDefNotFoundException.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.graphscope.groot.common.exception; - -public class PropertyDefNotFoundException extends GrootException { - public PropertyDefNotFoundException(String msg) { - super(msg); - } -} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/GraphSchemaException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/PropertyNotFoundException.java similarity index 56% rename from interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/GraphSchemaException.java rename to interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/PropertyNotFoundException.java index 90fb8b225386..ef6a66b40077 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/GraphSchemaException.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/PropertyNotFoundException.java @@ -15,20 +15,22 @@ */ package com.alibaba.graphscope.groot.common.exception; -public class GraphSchemaException extends GrootException { - public GraphSchemaException() { - super(); +import com.alibaba.graphscope.proto.Code; + +public class PropertyNotFoundException extends GrootException { + public PropertyNotFoundException() { + super(Code.PROPERTY_NOT_FOUND); } - public GraphSchemaException(Exception exception) { - super(exception); + public PropertyNotFoundException(Exception exception) { + super(Code.PROPERTY_NOT_FOUND, exception); } - public GraphSchemaException(String message) { - super(message); + public PropertyNotFoundException(String message) { + super(Code.PROPERTY_NOT_FOUND, message); } - public GraphSchemaException(String message, Exception exception) { - super(message, exception); + public PropertyNotFoundException(String message, Exception exception) { + super(Code.PROPERTY_NOT_FOUND, message, exception); } } diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/QueryFailedException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/QueryFailedException.java new file mode 100644 index 000000000000..117c941fbdec --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/QueryFailedException.java @@ -0,0 +1,21 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class QueryFailedException extends GrootException { + public QueryFailedException(Throwable t) { + super(Code.QUERY_FAILED, t); + } + + public QueryFailedException(String msg) { + super(Code.QUERY_FAILED, msg); + } + + public QueryFailedException(String msg, Throwable t) { + super(Code.QUERY_FAILED, msg, t); + } + + public QueryFailedException() { + super(Code.QUERY_FAILED); + } +} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/QueueRejectException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/QueueRejectException.java index 2f7b26f0ba94..ae509cc7fa4b 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/QueueRejectException.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/QueueRejectException.java @@ -15,10 +15,22 @@ */ package com.alibaba.graphscope.groot.common.exception; -import com.alibaba.graphscope.groot.common.util.ExceptionUtils; +import com.alibaba.graphscope.proto.Code; public class QueueRejectException extends GrootException { public QueueRejectException(String msg) { - super(ExceptionUtils.ErrorCode.QueueReject, msg); + super(Code.QUEUE_REJECT, msg); + } + + public QueueRejectException(String msg, Throwable t) { + super(Code.QUEUE_REJECT, msg, t); + } + + public QueueRejectException() { + super(Code.QUEUE_REJECT); + } + + public QueueRejectException(Throwable t) { + super(Code.QUEUE_REJECT, t); } } diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/RealtimeWriteFailedException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/RealtimeWriteFailedException.java new file mode 100644 index 000000000000..5ce8b96d56c7 --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/RealtimeWriteFailedException.java @@ -0,0 +1,21 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class RealtimeWriteFailedException extends GrootException { + public RealtimeWriteFailedException(Throwable t) { + super(Code.REALTIME_WRITE_FAILED, t); + } + + public RealtimeWriteFailedException(String msg) { + super(Code.REALTIME_WRITE_FAILED, msg); + } + + public RealtimeWriteFailedException(String msg, Throwable t) { + super(Code.REALTIME_WRITE_FAILED, msg, t); + } + + public RealtimeWriteFailedException() { + super(Code.REALTIME_WRITE_FAILED); + } +} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/ResourceExhaustedException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/ResourceExhaustedException.java new file mode 100644 index 000000000000..e8c9700df56c --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/ResourceExhaustedException.java @@ -0,0 +1,21 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class ResourceExhaustedException extends GrootException { + public ResourceExhaustedException(Throwable t) { + super(Code.RESOURCE_EXHAUSTED, t); + } + + public ResourceExhaustedException(String msg) { + super(Code.RESOURCE_EXHAUSTED, msg); + } + + public ResourceExhaustedException(String msg, Throwable t) { + super(Code.RESOURCE_EXHAUSTED, msg, t); + } + + public ResourceExhaustedException() { + super(Code.RESOURCE_EXHAUSTED); + } +} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/ServiceNotReadyException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/ServiceNotReadyException.java index 6e1f293f7676..e8f479626f59 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/ServiceNotReadyException.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/ServiceNotReadyException.java @@ -15,8 +15,22 @@ */ package com.alibaba.graphscope.groot.common.exception; +import com.alibaba.graphscope.proto.Code; + public class ServiceNotReadyException extends GrootException { public ServiceNotReadyException(String msg) { - super(msg); + super(Code.SERVICE_NOT_READY, msg); + } + + public ServiceNotReadyException(String msg, Throwable t) { + super(Code.SERVICE_NOT_READY, msg, t); + } + + public ServiceNotReadyException() { + super(Code.SERVICE_NOT_READY); + } + + public ServiceNotReadyException(Throwable t) { + super(Code.SERVICE_NOT_READY, t); } } diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/TimeoutException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/TimeoutException.java new file mode 100644 index 000000000000..604b64810b9e --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/TimeoutException.java @@ -0,0 +1,21 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class TimeoutException extends GrootException { + public TimeoutException(Throwable t) { + super(Code.TIMEOUT, t); + } + + public TimeoutException(String msg) { + super(Code.TIMEOUT, msg); + } + + public TimeoutException(String msg, Throwable t) { + super(Code.TIMEOUT, msg, t); + } + + public TimeoutException() { + super(Code.TIMEOUT); + } +} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/TooManyVersionsException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/TooManyVersionsException.java new file mode 100644 index 000000000000..4db9b136a571 --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/TooManyVersionsException.java @@ -0,0 +1,21 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class TooManyVersionsException extends GrootException { + public TooManyVersionsException(Throwable t) { + super(Code.TOO_MANY_VERSIONS, t); + } + + public TooManyVersionsException(String msg) { + super(Code.TOO_MANY_VERSIONS, msg); + } + + public TooManyVersionsException(String msg, Throwable t) { + super(Code.TOO_MANY_VERSIONS, msg, t); + } + + public TooManyVersionsException() { + super(Code.TOO_MANY_VERSIONS); + } +} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/TypeDefNotFoundException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/TypeDefNotFoundException.java deleted file mode 100644 index ce53886b679a..000000000000 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/TypeDefNotFoundException.java +++ /dev/null @@ -1,22 +0,0 @@ -/** - * Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.graphscope.groot.common.exception; - -public class TypeDefNotFoundException extends GrootException { - public TypeDefNotFoundException(String msg) { - super(msg); - } -} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/GraphElementNotFoundException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/TypeNotFoundException.java similarity index 60% rename from interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/GraphElementNotFoundException.java rename to interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/TypeNotFoundException.java index 13a7a178fcf8..4d04bd6294a2 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/GraphElementNotFoundException.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/TypeNotFoundException.java @@ -15,20 +15,23 @@ */ package com.alibaba.graphscope.groot.common.exception; -public class GraphElementNotFoundException extends GraphSchemaException { - public GraphElementNotFoundException() { - super(); +import com.alibaba.graphscope.proto.Code; + +public class TypeNotFoundException extends GrootException { + + public TypeNotFoundException(Throwable t) { + super(Code.TYPE_NOT_FOUND, t); } - public GraphElementNotFoundException(Exception exception) { - super(exception); + public TypeNotFoundException(String message) { + super(Code.TYPE_NOT_FOUND, message); } - public GraphElementNotFoundException(String message) { - super(message); + public TypeNotFoundException(String msg, Throwable t) { + super(Code.TYPE_NOT_FOUND, msg, t); } - public GraphElementNotFoundException(String message, Exception exception) { - super(message, exception); + public TypeNotFoundException() { + super(Code.TYPE_NOT_FOUND); } } diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/UnimplementedException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/UnimplementedException.java new file mode 100644 index 000000000000..536b1995cf25 --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/UnimplementedException.java @@ -0,0 +1,21 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class UnimplementedException extends GrootException { + public UnimplementedException(Throwable t) { + super(Code.UNIMPLEMENTED, t); + } + + public UnimplementedException(String msg) { + super(Code.UNIMPLEMENTED, msg); + } + + public UnimplementedException(String msg, Throwable t) { + super(Code.UNIMPLEMENTED, msg, t); + } + + public UnimplementedException() { + super(Code.UNIMPLEMENTED); + } +} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/UnsupportedOperationException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/UnsupportedOperationException.java new file mode 100644 index 000000000000..96c5bd10d4d5 --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/UnsupportedOperationException.java @@ -0,0 +1,21 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class UnsupportedOperationException extends GrootException { + public UnsupportedOperationException(Throwable t) { + super(Code.UNSUPPORTED_OPERATION, t); + } + + public UnsupportedOperationException(String msg) { + super(Code.UNSUPPORTED_OPERATION, msg); + } + + public UnsupportedOperationException(String msg, Throwable t) { + super(Code.UNSUPPORTED_OPERATION, msg, t); + } + + public UnsupportedOperationException() { + super(Code.UNSUPPORTED_OPERATION); + } +} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/ValueTypeMismatchException.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/ValueTypeMismatchException.java new file mode 100644 index 000000000000..e96981735938 --- /dev/null +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/exception/ValueTypeMismatchException.java @@ -0,0 +1,21 @@ +package com.alibaba.graphscope.groot.common.exception; + +import com.alibaba.graphscope.proto.Code; + +public class ValueTypeMismatchException extends GrootException { + public ValueTypeMismatchException(Throwable t) { + super(Code.VALUE_TYPE_MISMATCH, t); + } + + public ValueTypeMismatchException(String msg) { + super(Code.VALUE_TYPE_MISMATCH, msg); + } + + public ValueTypeMismatchException(String msg, Throwable t) { + super(Code.VALUE_TYPE_MISMATCH, msg, t); + } + + public ValueTypeMismatchException() { + super(Code.VALUE_TYPE_MISMATCH); + } +} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/meta/DataType.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/meta/DataType.java index 009bdabe6947..38680dc11d2b 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/meta/DataType.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/meta/DataType.java @@ -16,7 +16,8 @@ package com.alibaba.graphscope.groot.common.meta; import com.alibaba.graphscope.groot.common.exception.GrootException; -import com.alibaba.graphscope.groot.common.util.ExceptionUtils; +import com.alibaba.graphscope.groot.common.exception.InternalException; +import com.alibaba.graphscope.groot.common.exception.InvalidDataTypeException; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonValue; @@ -89,8 +90,7 @@ public void setExpression(String expression) throws GrootException { } if (!isValid(expression)) { - throw new GrootException( - ExceptionUtils.ErrorCode.DataTypeNotValid, + throw new InvalidDataTypeException( "expression is not valid, subType " + "must be primitiveTypes: " + InternalDataType.primitiveTypes); @@ -170,7 +170,7 @@ public int getFixedSize() { case DOUBLE: return 8; default: - throw new RuntimeException("unreachable!"); + throw new InternalException("unreachable!"); } } diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/meta/DataTypeDeserializer.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/meta/DataTypeDeserializer.java index 258fbb161a62..392eaeb09569 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/meta/DataTypeDeserializer.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/meta/DataTypeDeserializer.java @@ -16,13 +16,11 @@ package com.alibaba.graphscope.groot.common.meta; import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.groot.common.exception.InvalidDataException; import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.DeserializationContext; import com.fasterxml.jackson.databind.deser.std.StdDeserializer; -import java.io.IOException; - public class DataTypeDeserializer extends StdDeserializer { public DataTypeDeserializer() { this(null); @@ -33,11 +31,10 @@ public DataTypeDeserializer(Class vc) { } @Override - public DataType deserialize(JsonParser p, DeserializationContext ctxt) - throws IOException, JsonProcessingException { + public DataType deserialize(JsonParser p, DeserializationContext ctxt) throws GrootException { // todo: only support List and Set currently. - String data = p.getValueAsString(); try { + String data = p.getValueAsString(); if (data.startsWith("LIST<")) { DataType ret = new DataType(InternalDataType.LIST); ret.setExpression(data.substring("LIST<".length(), data.length() - 1)); @@ -52,8 +49,8 @@ public DataType deserialize(JsonParser p, DeserializationContext ctxt) DataType ret = new DataType(InternalDataType.valueOf(data.toUpperCase())); return ret; } - } catch (GrootException ex) { - throw new IOException(ex); + } catch (Exception ex) { + throw new InvalidDataException(ex); } return null; } diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/meta/DataTypeSerializer.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/meta/DataTypeSerializer.java index b95ec59a3efd..a24529c362c6 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/meta/DataTypeSerializer.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/meta/DataTypeSerializer.java @@ -15,6 +15,8 @@ */ package com.alibaba.graphscope.groot.common.meta; +import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.groot.common.exception.InvalidDataTypeException; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.SerializerProvider; import com.fasterxml.jackson.databind.ser.std.StdSerializer; @@ -29,7 +31,7 @@ public DataTypeSerializer() { @Override public void serialize(DataType value, JsonGenerator gen, SerializerProvider provider) - throws IOException { + throws GrootException, IOException { if (value.isPrimitiveType()) { gen.writeString(value.name()); } else { @@ -41,7 +43,8 @@ public void serialize(DataType value, JsonGenerator gen, SerializerProvider prov } else if (value.getType().equals(InternalDataType.MAP)) { prefix = "M"; } else { - throw new IOException(String.format("unknown data type: %s", value.toString())); + throw new InvalidDataTypeException( + String.format("unknown data type: %s", value.toString())); } gen.writeString(String.format("%s<%s>", prefix, value.getExpression())); } diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/api/GraphSchema.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/api/GraphSchema.java index d7b93df9ce77..c8b3c77d3dea 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/api/GraphSchema.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/api/GraphSchema.java @@ -15,24 +15,24 @@ */ package com.alibaba.graphscope.groot.common.schema.api; -import com.alibaba.graphscope.groot.common.exception.GraphElementNotFoundException; -import com.alibaba.graphscope.groot.common.exception.GraphPropertyNotFoundException; +import com.alibaba.graphscope.groot.common.exception.PropertyNotFoundException; +import com.alibaba.graphscope.groot.common.exception.TypeNotFoundException; import java.util.List; import java.util.Map; public interface GraphSchema { - GraphElement getElement(String label) throws GraphElementNotFoundException; + GraphElement getElement(String label) throws TypeNotFoundException; - GraphElement getElement(int labelId) throws GraphElementNotFoundException; + GraphElement getElement(int labelId) throws TypeNotFoundException; List getVertexList(); List getEdgeList(); - Integer getPropertyId(String propName) throws GraphPropertyNotFoundException; + Integer getPropertyId(String propName) throws PropertyNotFoundException; - String getPropertyName(int propId) throws GraphPropertyNotFoundException; + String getPropertyName(int propId) throws PropertyNotFoundException; Map getPropertyList(String propName); diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/impl/DefaultGraphEdge.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/impl/DefaultGraphEdge.java index ca0fb6831079..ab06f41f4ae4 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/impl/DefaultGraphEdge.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/impl/DefaultGraphEdge.java @@ -15,6 +15,7 @@ */ package com.alibaba.graphscope.groot.common.schema.impl; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.groot.common.schema.api.EdgeRelation; import com.alibaba.graphscope.groot.common.schema.api.GraphEdge; import com.alibaba.graphscope.groot.common.schema.api.GraphProperty; @@ -117,7 +118,7 @@ public GraphProperty getProperty(int propId) { } } - throw new IllegalArgumentException("Invalid property id " + propId); + throw new InvalidArgumentException("Invalid property id " + propId); } @Override @@ -128,7 +129,7 @@ public GraphProperty getProperty(String propName) { } } - throw new IllegalArgumentException("Invalid property name " + propName); + throw new InvalidArgumentException("Invalid property name " + propName); } @Override diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/impl/DefaultGraphSchema.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/impl/DefaultGraphSchema.java index 0ae122a383cd..c3b2ac3f4a0c 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/impl/DefaultGraphSchema.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/impl/DefaultGraphSchema.java @@ -15,8 +15,9 @@ */ package com.alibaba.graphscope.groot.common.schema.impl; -import com.alibaba.graphscope.groot.common.exception.GraphElementNotFoundException; -import com.alibaba.graphscope.groot.common.exception.GraphPropertyNotFoundException; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; +import com.alibaba.graphscope.groot.common.exception.PropertyNotFoundException; +import com.alibaba.graphscope.groot.common.exception.TypeNotFoundException; import com.alibaba.graphscope.groot.common.schema.api.EdgeRelation; import com.alibaba.graphscope.groot.common.schema.api.GraphEdge; import com.alibaba.graphscope.groot.common.schema.api.GraphElement; @@ -55,23 +56,23 @@ public DefaultGraphSchema( } @Override - public GraphElement getElement(String label) throws GraphElementNotFoundException { + public GraphElement getElement(String label) throws TypeNotFoundException { if (vertexList.containsKey(label)) { return vertexList.get(label); } else if (edgeList.containsKey(label)) { return edgeList.get(label); } - throw new GraphElementNotFoundException("label " + label + " not exist"); + throw new TypeNotFoundException("label " + label + " not exist"); } @Override - public GraphElement getElement(int labelId) throws GraphElementNotFoundException { + public GraphElement getElement(int labelId) throws TypeNotFoundException { if (idToLabelList.containsKey(labelId)) { return getElement(idToLabelList.get(labelId)); } - throw new GraphElementNotFoundException("label not exist for label ID " + labelId); + throw new TypeNotFoundException("label not exist for label ID " + labelId); } @Override @@ -85,22 +86,22 @@ public List getEdgeList() { } @Override - public Integer getPropertyId(String propName) throws GraphPropertyNotFoundException { + public Integer getPropertyId(String propName) throws PropertyNotFoundException { if (propNameToIdList.containsKey(propName)) { return propNameToIdList.get(propName); } - throw new GraphPropertyNotFoundException("property " + propName + " not exist"); + throw new PropertyNotFoundException("property " + propName + " not exist"); } @Override - public String getPropertyName(int propId) throws GraphPropertyNotFoundException { + public String getPropertyName(int propId) throws PropertyNotFoundException { for (Map.Entry entry : propNameToIdList.entrySet()) { if (entry.getValue() == propId) { return entry.getKey(); } } - throw new GraphPropertyNotFoundException("property not exist for property id " + propId); + throw new PropertyNotFoundException("property not exist for property id " + propId); } @Override @@ -223,7 +224,7 @@ public static GraphSchema buildSchemaFromJson(String schemaJson) { return new DefaultGraphSchema(vertexList, edgeList, propNameToIdList); } catch (JsonProcessingException e) { - throw new RuntimeException(e); + throw new InvalidArgumentException(e); } } diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/impl/DefaultGraphVertex.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/impl/DefaultGraphVertex.java index 483f139610eb..0085f0fa5bbc 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/impl/DefaultGraphVertex.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/impl/DefaultGraphVertex.java @@ -15,6 +15,7 @@ */ package com.alibaba.graphscope.groot.common.schema.impl; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.groot.common.schema.api.GraphProperty; import com.alibaba.graphscope.groot.common.schema.api.GraphVertex; import com.alibaba.graphscope.groot.common.schema.wrapper.TypeDef; @@ -104,7 +105,7 @@ public GraphProperty getProperty(int propId) { return graphProperty; } } - throw new IllegalArgumentException("Can't get graph property for id " + propId); + throw new InvalidArgumentException("Can't get graph property for id " + propId); } @Override @@ -114,7 +115,7 @@ public GraphProperty getProperty(String propName) { return graphProperty; } } - throw new IllegalArgumentException("Can't get graph property for name " + propName); + throw new InvalidArgumentException("Can't get graph property for name " + propName); } @Override diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/mapper/EdgeTypeMapper.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/mapper/EdgeTypeMapper.java index 4e42b63006af..1848370be956 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/mapper/EdgeTypeMapper.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/mapper/EdgeTypeMapper.java @@ -13,6 +13,7 @@ */ package com.alibaba.graphscope.groot.common.schema.mapper; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.groot.common.schema.api.EdgeRelation; import com.alibaba.graphscope.groot.common.schema.api.GraphEdge; import com.alibaba.graphscope.groot.common.schema.api.GraphProperty; @@ -93,7 +94,7 @@ public GraphEdge toEdgeType(Map vertexTypeMap) { List primaryKeyList = new ArrayList<>(); if (indexes != null && indexes.size() > 0) { if (indexes.size() > 1) { - throw new IllegalArgumentException( + throw new InvalidArgumentException( "Only support primary key now for " + this.indexes); } primaryKeyList = indexes.get(0).getPropertyNames(); diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/mapper/GraphSchemaMapper.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/mapper/GraphSchemaMapper.java index f28a2f0f57d5..febb0f953150 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/mapper/GraphSchemaMapper.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/mapper/GraphSchemaMapper.java @@ -13,6 +13,7 @@ */ package com.alibaba.graphscope.groot.common.schema.mapper; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.groot.common.schema.api.GraphEdge; import com.alibaba.graphscope.groot.common.schema.api.GraphSchema; import com.alibaba.graphscope.groot.common.schema.api.GraphVertex; @@ -70,7 +71,7 @@ public String toJsonString() { try { return objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(this); } catch (JsonProcessingException e) { - throw new IllegalArgumentException("convert graph schema to json failed", e); + throw new InvalidArgumentException("convert graph schema to json failed", e); } } @@ -113,7 +114,7 @@ public static GraphSchemaMapper parseFromJson(String schemaJson) { } return graphSchema; } catch (JsonProcessingException e) { - throw new RuntimeException(e); + throw new InvalidArgumentException(e); } } diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/mapper/VertexTypeMapper.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/mapper/VertexTypeMapper.java index cd3d2cd2fe7c..b29408c012fb 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/mapper/VertexTypeMapper.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/mapper/VertexTypeMapper.java @@ -13,6 +13,7 @@ */ package com.alibaba.graphscope.groot.common.schema.mapper; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.groot.common.schema.api.GraphProperty; import com.alibaba.graphscope.groot.common.schema.api.GraphVertex; import com.alibaba.graphscope.groot.common.schema.impl.DefaultGraphVertex; @@ -74,7 +75,7 @@ public GraphVertex toVertexType() { .collect(Collectors.toList()); /// TODO only support primary key now if (this.indexes.size() > 1) { - throw new IllegalArgumentException("Only support primary key now for " + this.indexes); + throw new InvalidArgumentException("Only support primary key now for " + this.indexes); } List primaryKeyList = indexes.get(0).getPropertyNames(); diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/unified/Graph.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/unified/Graph.java index 4578f637da05..1015e3149350 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/unified/Graph.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/unified/Graph.java @@ -1,7 +1,7 @@ package com.alibaba.graphscope.groot.common.schema.unified; -import com.alibaba.graphscope.groot.common.exception.GraphElementNotFoundException; -import com.alibaba.graphscope.groot.common.exception.GraphPropertyNotFoundException; +import com.alibaba.graphscope.groot.common.exception.PropertyNotFoundException; +import com.alibaba.graphscope.groot.common.exception.TypeNotFoundException; import com.alibaba.graphscope.groot.common.schema.api.*; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.core.JsonProcessingException; @@ -39,7 +39,7 @@ public String toString() { } @Override - public GraphElement getElement(String label) throws GraphElementNotFoundException { + public GraphElement getElement(String label) throws TypeNotFoundException { for (Type type : schema.vertexTypes) { if (Objects.equals(type.getLabel(), label)) { return type; @@ -54,7 +54,7 @@ public GraphElement getElement(String label) throws GraphElementNotFoundExceptio } @Override - public GraphElement getElement(int labelId) throws GraphElementNotFoundException { + public GraphElement getElement(int labelId) throws TypeNotFoundException { for (Type type : schema.vertexTypes) { if (type.getLabelId() == labelId) { return type; @@ -81,7 +81,7 @@ public List getEdgeList() { } @Override - public Integer getPropertyId(String propName) throws GraphPropertyNotFoundException { + public Integer getPropertyId(String propName) throws PropertyNotFoundException { for (Type type : schema.vertexTypes) { for (GraphProperty property : type.getPropertyList()) { if (Objects.equals(property.getName(), propName)) { @@ -100,7 +100,7 @@ public Integer getPropertyId(String propName) throws GraphPropertyNotFoundExcept } @Override - public String getPropertyName(int propId) throws GraphPropertyNotFoundException { + public String getPropertyName(int propId) throws PropertyNotFoundException { for (Type type : schema.vertexTypes) { for (GraphProperty property : type.getPropertyList()) { if (Objects.equals(property.getId(), propId)) { diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/unified/PropertyType.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/unified/PropertyType.java index 9ca73279d900..a49a0cd0e4f1 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/unified/PropertyType.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/unified/PropertyType.java @@ -1,5 +1,6 @@ package com.alibaba.graphscope.groot.common.schema.unified; +import com.alibaba.graphscope.groot.common.exception.InvalidDataTypeException; import com.alibaba.graphscope.groot.common.schema.wrapper.DataType; public class PropertyType { @@ -39,6 +40,6 @@ public DataType toImpl() { return DataType.DOUBLE; } } - throw new RuntimeException("Cannot parse propertyType " + this); + throw new InvalidDataTypeException("Cannot parse propertyType " + this); } } diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/unified/Schema.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/unified/Schema.java index d2807a2485f6..9b67f9b8f8d3 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/unified/Schema.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/unified/Schema.java @@ -1,7 +1,6 @@ package com.alibaba.graphscope.groot.common.schema.unified; -import com.alibaba.graphscope.groot.common.exception.GraphElementNotFoundException; -import com.alibaba.graphscope.groot.common.schema.api.*; +import com.alibaba.graphscope.groot.common.exception.TypeNotFoundException; import java.util.List; import java.util.Objects; @@ -15,7 +14,7 @@ public String toString() { return "Schema{" + "vertexTypes=" + vertexTypes + ", edgeTypes=" + edgeTypes + '}'; } - private VertexType getVertexType(String label) throws GraphElementNotFoundException { + private VertexType getVertexType(String label) throws TypeNotFoundException { for (VertexType type : vertexTypes) { if (Objects.equals(type.getLabel(), label)) { return type; diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/wrapper/DataType.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/wrapper/DataType.java index 3dbfe3920375..a2efedb712de 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/wrapper/DataType.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/wrapper/DataType.java @@ -15,6 +15,8 @@ */ package com.alibaba.graphscope.groot.common.schema.wrapper; +import com.alibaba.graphscope.groot.common.exception.InvalidDataTypeException; +import com.alibaba.graphscope.groot.common.exception.UnsupportedOperationException; import com.alibaba.graphscope.groot.common.meta.InternalDataType; import com.alibaba.graphscope.proto.groot.DataTypePb; @@ -49,7 +51,7 @@ public enum DataType { public static DataType fromId(byte id) { if (id < 0 || id >= TYPES.length) { - throw new IllegalArgumentException("Unknown DataType: [" + id + "]"); + throw new InvalidDataTypeException("Unknown DataType: [" + id + "]"); } return TYPES[id]; } @@ -182,14 +184,14 @@ public static DataType parseFromDataType( } default: { - throw new IllegalArgumentException( + throw new InvalidDataTypeException( "Unsupported property data type " + dataType); } } } default: { - throw new IllegalArgumentException( + throw new InvalidDataTypeException( "Unsupported property data type " + dataType); } } diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/wrapper/GraphDef.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/wrapper/GraphDef.java index b304d180567b..ecf88536a1d6 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/wrapper/GraphDef.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/wrapper/GraphDef.java @@ -13,10 +13,7 @@ */ package com.alibaba.graphscope.groot.common.schema.wrapper; -import com.alibaba.graphscope.groot.common.exception.GraphElementNotFoundException; -import com.alibaba.graphscope.groot.common.exception.GraphPropertyNotFoundException; -import com.alibaba.graphscope.groot.common.exception.InvalidSchemaException; -import com.alibaba.graphscope.groot.common.exception.TypeDefNotFoundException; +import com.alibaba.graphscope.groot.common.exception.*; import com.alibaba.graphscope.groot.common.schema.api.*; import com.alibaba.graphscope.groot.common.schema.impl.DefaultEdgeRelation; import com.alibaba.graphscope.groot.common.schema.impl.DefaultGraphEdge; @@ -101,7 +98,7 @@ private DefaultGraphEdge getEdgeType(TypeDef typeDef) { GraphVertex dstGraphVertex = this.vertexTypes.get(edgeKind.getDstVertexLabelId()); Long tableId = edgeTableIds.get(edgeKind); if (tableId == null) { - throw new IllegalArgumentException("no valid table id for [" + edgeKind + "]"); + throw new InvalidArgumentException("no valid table id for [" + edgeKind + "]"); } edgeRelations.add(new DefaultEdgeRelation(srcGraphVertex, dstGraphVertex, tableId)); } @@ -124,7 +121,7 @@ private DefaultGraphVertex getVertexType(TypeDef typeDef) { LabelId labelId = typeDef.getTypeLabelId(); Long tableId = vertexTableIds.get(labelId); if (tableId == null) { - throw new InvalidSchemaException("no valid table id for [" + typeDef.getLabel() + "]"); + throw new IllegalSchemaException("no valid table id for [" + typeDef.getLabel() + "]"); } return new DefaultGraphVertex(typeDef, tableId); } @@ -224,7 +221,7 @@ public GraphDefPb toProto() { public TypeDef getTypeDef(String label) { LabelId labelId = this.labelToId.get(label); if (labelId == null) { - throw new TypeDefNotFoundException("no such label [" + label + "]"); + throw new NotFoundException("no such label [" + label + "]"); } return getTypeDef(labelId); } @@ -250,30 +247,30 @@ public boolean hasEdgeKind(EdgeKind edgeKind) { } @Override - public GraphElement getElement(String label) throws GraphElementNotFoundException { + public GraphElement getElement(String label) throws TypeNotFoundException { LabelId labelId = labelToId.get(label); if (labelId == null) { - throw new GraphElementNotFoundException("schema element not found for label " + label); + throw new TypeNotFoundException("schema element not found for label " + label); } return convertVertexEdgeType(idToType.get(labelId)); } @Override - public GraphElement getElement(int labelId) throws GraphElementNotFoundException { + public GraphElement getElement(int labelId) throws TypeNotFoundException { TypeDef typeDef = idToType.get(new LabelId(labelId)); return convertVertexEdgeType(typeDef); } private GraphElement convertVertexEdgeType(TypeDef typeDef) { if (null == typeDef) { - throw new RuntimeException("No type def with given label id/name"); + throw new InvalidArgumentException("No type def with given label id/name"); } if (typeDef.getTypeEnum() == TypeEnum.VERTEX) { return getVertexType(typeDef); } else if (typeDef.getTypeEnum() == TypeEnum.EDGE) { return getEdgeType(typeDef); } else { - throw new IllegalArgumentException("Not support type value " + typeDef.getTypeEnum()); + throw new InvalidArgumentException("Not support type value " + typeDef.getTypeEnum()); } } @@ -288,23 +285,22 @@ public List getEdgeList() { } @Override - public Integer getPropertyId(String propertyName) throws GraphPropertyNotFoundException { + public Integer getPropertyId(String propertyName) throws PropertyNotFoundException { if (propertyNameToId.containsKey(propertyName)) { return propertyNameToId.get(propertyName); } - throw new GraphPropertyNotFoundException("property " + propertyName + " not exist"); + throw new PropertyNotFoundException("property " + propertyName + " not exist"); } @Override - public String getPropertyName(int propertyId) throws GraphPropertyNotFoundException { + public String getPropertyName(int propertyId) throws PropertyNotFoundException { for (Map.Entry entry : propertyNameToId.entrySet()) { if (entry.getValue() == propertyId) { return entry.getKey(); } } - throw new GraphPropertyNotFoundException( - "property not exist for property id " + propertyId); + throw new PropertyNotFoundException("property not exist for property id " + propertyId); } @Override diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/wrapper/PropertyValue.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/wrapper/PropertyValue.java index 8a7bff731556..0ee6eda7e8b7 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/wrapper/PropertyValue.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/wrapper/PropertyValue.java @@ -13,6 +13,8 @@ */ package com.alibaba.graphscope.groot.common.schema.wrapper; +import com.alibaba.graphscope.groot.common.exception.IllegalStateException; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.proto.groot.PropertyValuePb; import com.google.protobuf.ByteString; @@ -64,7 +66,7 @@ private static Object stringToObj(DataType dataType, Object val) { throw new IllegalStateException("Unexpected value: " + dataType); } } catch (Exception e) { - throw new IllegalArgumentException( + throw new InvalidArgumentException( "unable to parse object to bytes. DataType [" + dataType + "], Object [" diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/wrapper/SerdeUtils.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/wrapper/SerdeUtils.java index b08c9646bb66..f26007ead022 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/wrapper/SerdeUtils.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/wrapper/SerdeUtils.java @@ -13,6 +13,9 @@ */ package com.alibaba.graphscope.groot.common.schema.wrapper; +import com.alibaba.graphscope.groot.common.exception.IllegalStateException; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; + import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; @@ -73,7 +76,7 @@ public static byte[] objectToBytes(DataType dataType, Object valObject) { try { dos.writeInt(e); } catch (IOException ex) { - throw new IllegalArgumentException("write to bytes failed", ex); + throw new InvalidArgumentException("write to bytes failed", ex); } return null; }); @@ -89,7 +92,7 @@ public static byte[] objectToBytes(DataType dataType, Object valObject) { try { dos.writeLong(e); } catch (IOException ex) { - throw new IllegalArgumentException("write to bytes failed", ex); + throw new InvalidArgumentException("write to bytes failed", ex); } return null; }); @@ -105,7 +108,7 @@ public static byte[] objectToBytes(DataType dataType, Object valObject) { try { dos.writeFloat(e); } catch (IOException ex) { - throw new IllegalArgumentException("write to bytes failed", ex); + throw new InvalidArgumentException("write to bytes failed", ex); } return null; }); @@ -121,7 +124,7 @@ public static byte[] objectToBytes(DataType dataType, Object valObject) { try { dos.writeDouble(e); } catch (IOException ex) { - throw new IllegalArgumentException("write to bytes failed", ex); + throw new InvalidArgumentException("write to bytes failed", ex); } return null; }); @@ -149,7 +152,7 @@ public static byte[] objectToBytes(DataType dataType, Object valObject) { throw new IllegalStateException("Unexpected value: " + dataType); } } catch (Exception e) { - throw new IllegalArgumentException( + throw new InvalidArgumentException( "unable to parse object to bytes. DataType [" + dataType + "], Object [" @@ -200,7 +203,7 @@ public static Object bytesToObject(DataType dataType, byte[] valBytes) { try { return dis.readInt(); } catch (IOException e) { - throw new IllegalArgumentException( + throw new InvalidArgumentException( "parse val failed", e); } }); @@ -213,7 +216,7 @@ public static Object bytesToObject(DataType dataType, byte[] valBytes) { try { return dis.readLong(); } catch (IOException e) { - throw new IllegalArgumentException( + throw new InvalidArgumentException( "parse val failed", e); } }); @@ -226,7 +229,7 @@ public static Object bytesToObject(DataType dataType, byte[] valBytes) { try { return dis.readFloat(); } catch (IOException e) { - throw new IllegalArgumentException( + throw new InvalidArgumentException( "parse val failed", e); } }); @@ -239,7 +242,7 @@ public static Object bytesToObject(DataType dataType, byte[] valBytes) { try { return dis.readDouble(); } catch (IOException e) { - throw new IllegalArgumentException( + throw new InvalidArgumentException( "parse val failed", e); } }); @@ -267,7 +270,7 @@ public static Object bytesToObject(DataType dataType, byte[] valBytes) { } return valObject; } catch (IOException e) { - throw new IllegalArgumentException("parse val failed", e); + throw new InvalidArgumentException("parse val failed", e); } } diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/wrapper/TypeEnum.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/wrapper/TypeEnum.java index 7baa9e109f9d..b7f80a5b162c 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/wrapper/TypeEnum.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/schema/wrapper/TypeEnum.java @@ -13,6 +13,7 @@ */ package com.alibaba.graphscope.groot.common.schema.wrapper; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.proto.groot.TypeEnumPb; public enum TypeEnum { @@ -30,7 +31,7 @@ public enum TypeEnum { public static TypeEnum fromId(byte id) { if (id < 0 || id >= ALL.length) { - throw new IllegalArgumentException("Unknown TypeEnum: [" + id + "]"); + throw new InvalidArgumentException("Unknown TypeEnum: [" + id + "]"); } return ALL[id]; } diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/util/ExceptionUtils.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/util/ExceptionUtils.java deleted file mode 100644 index 4db49b329f59..000000000000 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/util/ExceptionUtils.java +++ /dev/null @@ -1,148 +0,0 @@ -/** - * Copyright 2020 Alibaba Group Holding Limited. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.graphscope.groot.common.util; - -import com.alibaba.graphscope.groot.common.exception.GrootException; -import com.alibaba.graphscope.groot.common.exception.QueueRejectException; - -public class ExceptionUtils { - - public enum ErrorCode { - OK(0), - PropertyAlreadyExist(1000), - PropertyNotExist(1001), - TypeAlreadyExist(1002), - TypeNotFound(1003), - RelationShipExistWithType(1004), - RelationShipAlreadyExist(1005), - PropertyNameAlreadyExist(1006), - UnSupportedMetaOperation(1007), - PropertyExistInType(1008), - SchemaPersistentError(1009), - DataTypeNotValid(1010), - DefaultValueNotMatchDataType(1011), - DefaultValueNotSupportedForDataType(1012), - RelationShipNotExist(1013), - InvalidTypeChanged(1014), - - IndexCanBeUsedOnlyOnEdge(1100), - IndexTypeMustUnique(1101), - - // client-frontend related - IllegalSession(2000), - FrontendServiceBusy(2001), - SessionTimeout(2002), - RealtimeWriteFailed(2003), - ServiceNotReady(2004), - RealtimeDataNotValid(2005), - QueueReject(2006), - - JobNotExist(3000), - JobIsTerminated(3001), - JobIsAlreadyInSnapshot(3002), - JobIdIsAlreadyExistInGraph(3003), - JobIdMustGreaterThanBefore(3004), - JobQueueIsFullForThisType(3005), - - SnapshotPersistentError(4000), - - // bulk load related - IllegalVertexLabel(5000), - IllegalEdgeLabel(5001), - IllegalSrcDstLabel(5002), - IllegalPlatform(5003), - IllegalStoreType(5004), - IllegalPrimaryKey(5005), - IllegalProperty(5006), - IllegalParameter(5007), - BuildDataFailed(5008), - OnlineDataFailed(5009), - SubmitBuildJobFailed(5010), - CancelBuildFailed(5011), - CancelOnlineFailed(5012), - CancelJobFailed(5013), - HttpRequestFailed(5014), - - // frontend related - AuthenticationFailed(6000), - AuthorizationFailed(6001), - - // studio related - @Deprecated - AkAuthenticatedFailed(7000), - - UnsupportedFilter(7001), - InstanceNotFound(7002), - DeleteInstanceFailed(7003), - InvalidParams(7004), - CannotRename(7005), - ClusterNotFound(7006), - AddPackageVersionFailed(7007), - InstanceNameExist(7008), - OneAdminNeededByInstance(7009), - MonitorClusterFailed(7010), - MonitorInstanceFailed(7011), - ApiNoPermission(7012), - CreateVpcAlbFailed(7013), - DeleteVpcAlbFailed(7014), - NoAvailablePackage(7015), - AkAccessKeyNotFound(7016), - AkSignatureFailed(7017), - AkSignatureMethodUnSupported(7018), - AkTimestampIllegal(7019), - AkSignatureNotFound(7020), - AkParameterInvalid(7021), - AkClientNotConnected(7022), - InvalidInstanceName(7023), - ResourceInsufficient(7024), - - Unknown(9999); - - private final int value; - - ErrorCode(int value) { - this.value = value; - } - - public int toInt() { - return value; - } - - public static ErrorCode fromInt(int code) { - for (ErrorCode errorCode : ErrorCode.class.getEnumConstants()) { - if (errorCode.toInt() == code) { - return errorCode; - } - } - - return ErrorCode.Unknown; - } - } - - public static void checkAndThrow(int errCode, String errMsg) throws GrootException { - ErrorCode errorCode = ErrorCode.fromInt(errCode); - switch (errorCode) { - case OK: - return; - case Unknown: - throw new RuntimeException(errMsg); - case QueueReject: - throw new QueueRejectException(errMsg); - default: - throw new GrootException(errorCode, errMsg); - } - } -} diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/util/IrSchemaParser.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/util/IrSchemaParser.java index d3ef0202aeb9..ba3245eed174 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/util/IrSchemaParser.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/util/IrSchemaParser.java @@ -16,6 +16,7 @@ package com.alibaba.graphscope.groot.common.util; +import com.alibaba.graphscope.groot.common.exception.UnsupportedOperationException; import com.alibaba.graphscope.groot.common.schema.api.*; import com.alibaba.graphscope.groot.common.schema.wrapper.DataType; import com.google.common.collect.ImmutableMap; diff --git a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/util/JSON.java b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/util/JSON.java index 4e3c09510b38..482954386512 100644 --- a/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/util/JSON.java +++ b/interactive_engine/common/src/main/java/com/alibaba/graphscope/groot/common/util/JSON.java @@ -15,7 +15,7 @@ */ package com.alibaba.graphscope.groot.common.util; -import com.alibaba.graphscope.groot.common.exception.JSONException; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.*; @@ -25,9 +25,6 @@ import java.io.IOException; import java.util.List; -/** - * Created by wubincen on 2018/5/3 - */ public class JSON { private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); @@ -43,7 +40,7 @@ public static String toJson(Object object) { try { return OBJECT_MAPPER.writeValueAsString(object); } catch (JsonProcessingException e) { - throw new JSONException("JsonProcessingException : ", e); + throw new InvalidArgumentException("JsonProcessingException : ", e); } } @@ -51,7 +48,7 @@ public static T fromJson(String json, TypeReference typeRef) throws Runti try { return (json == null || json.isEmpty()) ? null : OBJECT_MAPPER.readValue(json, typeRef); } catch (Exception e) { - throw new RuntimeException(e); + throw new InvalidArgumentException(e); } } @@ -59,7 +56,7 @@ public static T fromJson(String json, Class valueType) { try { return OBJECT_MAPPER.readValue(json, valueType); } catch (IOException e) { - throw new JSONException("IOException : ", e); + throw new InvalidArgumentException("IOException : ", e); } } @@ -67,7 +64,7 @@ public static JsonNode fromJson(String json) { try { return OBJECT_MAPPER.readTree(json); } catch (IOException e) { - throw new JSONException("IOException : ", e); + throw new InvalidArgumentException("IOException : ", e); } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GraphOptSchema.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GraphOptSchema.java index 1ec360b5b7e4..6b729d6d6e0d 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GraphOptSchema.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/GraphOptSchema.java @@ -16,7 +16,7 @@ package com.alibaba.graphscope.common.ir.meta.schema; -import com.alibaba.graphscope.groot.common.exception.GraphElementNotFoundException; +import com.alibaba.graphscope.groot.common.exception.TypeNotFoundException; import com.alibaba.graphscope.groot.common.schema.api.GraphElement; import org.apache.calcite.plan.RelOptCluster; @@ -57,7 +57,7 @@ public RelOptTable getTableForMember(List tableName) { try { GraphElement element = rootSchema.getElement(labelName); return createRelOptTable(tableName, element); - } catch (GraphElementNotFoundException e) { + } catch (TypeNotFoundException e) { throw Static.RESOURCE.tableNotFound(labelName).ex(); } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrGraphSchema.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrGraphSchema.java index 2a288da38e7a..351ad63a4c6a 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrGraphSchema.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/common/ir/meta/schema/IrGraphSchema.java @@ -16,8 +16,8 @@ package com.alibaba.graphscope.common.ir.meta.schema; -import com.alibaba.graphscope.groot.common.exception.GraphElementNotFoundException; -import com.alibaba.graphscope.groot.common.exception.GraphPropertyNotFoundException; +import com.alibaba.graphscope.groot.common.exception.PropertyNotFoundException; +import com.alibaba.graphscope.groot.common.exception.TypeNotFoundException; import com.alibaba.graphscope.groot.common.schema.api.*; import com.alibaba.graphscope.groot.common.util.IrSchemaParser; @@ -68,12 +68,12 @@ public String schemaJson() { } @Override - public GraphElement getElement(String s) throws GraphElementNotFoundException { + public GraphElement getElement(String s) throws TypeNotFoundException { return this.graphSchema.getElement(s); } @Override - public GraphElement getElement(int i) throws GraphElementNotFoundException { + public GraphElement getElement(int i) throws TypeNotFoundException { return this.graphSchema.getElement(i); } @@ -88,12 +88,12 @@ public List getEdgeList() { } @Override - public Integer getPropertyId(String s) throws GraphPropertyNotFoundException { + public Integer getPropertyId(String s) throws PropertyNotFoundException { return this.graphSchema.getPropertyId(s); } @Override - public String getPropertyName(int i) throws GraphPropertyNotFoundException { + public String getPropertyName(int i) throws PropertyNotFoundException { return this.graphSchema.getPropertyName(i); } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/integration/suite/graphAlgo/GraphAlgoQueries.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/integration/suite/graphAlgo/GraphAlgoQueries.java index 79e0c94024c4..8fa2aa631468 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/integration/suite/graphAlgo/GraphAlgoQueries.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/cypher/integration/suite/graphAlgo/GraphAlgoQueries.java @@ -179,16 +179,15 @@ public static QueryContext get_graph_algo_test11() { public static QueryContext get_graph_algo_test12() { String query = "MATCH (p:Paper)-[:WorkOn]->(a:Task),(a)-[:Belong]->(t: Topic)\n" - + " RETURN t.topic,COUNT(p);"; + + "RETURN t.topic as topic, COUNT(p) as count ORDER BY count DESC," + + " topic DESC LIMIT 5;"; List expected = Arrays.asList( - "Record<{topic: \"Centrality\", $f1: 18}>", - "Record<{topic: \"Community Detection\", $f1: 13}>", - "Record<{topic: \"Similarity\", $f1: 7}>", - "Record<{topic: \"Cohesive Subgraph\", $f1: 12}>", - "Record<{topic: \"Pattern Matching\", $f1: 30}>", - "Record<{topic: \"Traversal\", $f1: 29}>", - "Record<{topic: \"Covering\", $f1: 14}>"); + "Record<{topic: \"Pattern Matching\", count: 30}>", + "Record<{topic: \"Traversal\", count: 29}>", + "Record<{topic: \"Centrality\", count: 18}>", + "Record<{topic: \"Covering\", count: 14}>", + "Record<{topic: \"Community Detection\", count: 13}>"); return new QueryContext(query, expected); } @@ -197,47 +196,49 @@ public static QueryContext get_graph_algo_test13() { "MATCH (t: Topic)<-[:Belong]-(a:Task),\n" + " (a)<-[:WorkOn]-(p:Paper)-[:Use]->(s:Solution),\n" + " (s)-[:ApplyOn]->(ch:Challenge)\n" - + " RETURN t.topic, ch.challenge, COUNT(p);"; + + " RETURN t.topic as topic, ch.challenge as challenge, COUNT(p) as count" + + " ORDER BY count DESC, topic ASC, challenge ASC;"; List expected = Arrays.asList( - "Record<{topic: \"Centrality\", challenge: \"Parallelism\", $f2: 11}>", - "Record<{topic: \"Centrality\", challenge: \"Bandwidth\", $f2: 5}>", + "Record<{topic: \"Pattern Matching\", challenge: \"Communication" + + " Overhead\", count: 22}>", + "Record<{topic: \"Pattern Matching\", challenge: \"Load Balance\"," + + " count: 16}>", + "Record<{topic: \"Traversal\", challenge: \"Communication Overhead\"," + + " count: 13}>", + "Record<{topic: \"Traversal\", challenge: \"Parallelism\", count: 12}>", + "Record<{topic: \"Centrality\", challenge: \"Parallelism\", count: 11}>", "Record<{topic: \"Centrality\", challenge: \"Communication Overhead\"," - + " $f2: 9}>", - "Record<{topic: \"Centrality\", challenge: \"Load Balance\", $f2: 4}>", + + " count: 9}>", + "Record<{topic: \"Cohesive Subgraph\", challenge: \"Communication" + + " Overhead\", count: 9}>", "Record<{topic: \"Community Detection\", challenge: \"Communication" - + " Overhead\", $f2: 9}>", - "Record<{topic: \"Community Detection\", challenge: \"Load Balance\"," - + " $f2: 4}>", - "Record<{topic: \"Community Detection\", challenge: \"Parallelism\"," - + " $f2: 4}>", + + " Overhead\", count: 9}>", + "Record<{topic: \"Covering\", challenge: \"Parallelism\", count: 9}>", + "Record<{topic: \"Traversal\", challenge: \"Load Balance\", count: 8}>", + "Record<{topic: \"Cohesive Subgraph\", challenge: \"Parallelism\"," + + " count: 7}>", "Record<{topic: \"Similarity\", challenge: \"Communication Overhead\"," - + " $f2: 7}>", - "Record<{topic: \"Similarity\", challenge: \"Parallelism\", $f2: 2}>", - "Record<{topic: \"Similarity\", challenge: \"Load Balance\", $f2: 1}>", - "Record<{topic: \"Cohesive Subgraph\", challenge: \"Parallelism\", $f2:" - + " 7}>", - "Record<{topic: \"Cohesive Subgraph\", challenge: \"Communication" - + " Overhead\", $f2: 9}>", - "Record<{topic: \"Cohesive Subgraph\", challenge: \"Load Balance\", $f2:" - + " 4}>", - "Record<{topic: \"Pattern Matching\", challenge: \"Load Balance\", $f2:" - + " 16}>", - "Record<{topic: \"Pattern Matching\", challenge: \"Communication" - + " Overhead\", $f2: 22}>", - "Record<{topic: \"Pattern Matching\", challenge: \"Parallelism\", $f2:" - + " 1}>", - "Record<{topic: \"Pattern Matching\", challenge: \"Bandwidth\", $f2:" + + " count: 7}>", + "Record<{topic: \"Pattern Matching\", challenge: \"Bandwidth\", count:" + " 6}>", - "Record<{topic: \"Traversal\", challenge: \"Parallelism\", $f2: 12}>", - "Record<{topic: \"Traversal\", challenge: \"Communication Overhead\"," - + " $f2: 13}>", - "Record<{topic: \"Traversal\", challenge: \"Load Balance\", $f2: 8}>", - "Record<{topic: \"Traversal\", challenge: \"Bandwidth\", $f2: 2}>", - "Record<{topic: \"Covering\", challenge: \"Parallelism\", $f2: 9}>", + "Record<{topic: \"Centrality\", challenge: \"Bandwidth\", count: 5}>", + "Record<{topic: \"Centrality\", challenge: \"Load Balance\", count: 4}>", + "Record<{topic: \"Cohesive Subgraph\", challenge: \"Load Balance\"," + + " count: 4}>", + "Record<{topic: \"Community Detection\", challenge: \"Load Balance\"," + + " count: 4}>", + "Record<{topic: \"Community Detection\", challenge: \"Parallelism\"," + + " count: 4}>", "Record<{topic: \"Covering\", challenge: \"Communication Overhead\"," - + " $f2: 4}>", - "Record<{topic: \"Covering\", challenge: \"Load Balance\", $f2: 1}>"); + + " count: 4}>", + "Record<{topic: \"Similarity\", challenge: \"Parallelism\", count: 2}>", + "Record<{topic: \"Traversal\", challenge: \"Bandwidth\", count: 2}>", + "Record<{topic: \"Covering\", challenge: \"Load Balance\", count: 1}>", + "Record<{topic: \"Pattern Matching\", challenge: \"Parallelism\", count:" + + " 1}>", + "Record<{topic: \"Similarity\", challenge: \"Load Balance\", count:" + + " 1}>"); return new QueryContext(query, expected); } } diff --git a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/LifeCycleSupplier.java b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/LifeCycleSupplier.java index 79691bc64cca..3a46c6fa408c 100644 --- a/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/LifeCycleSupplier.java +++ b/interactive_engine/compiler/src/main/java/com/alibaba/graphscope/gremlin/plugin/processor/LifeCycleSupplier.java @@ -91,7 +91,7 @@ public GremlinExecutor.LifeCycle get() { GraphPlanner.Summary summary = value.summary; statusCallback .getQueryLogger() - .info("ir plan {}", summary.getPhysicalPlan().explain()); + .debug("ir plan {}", summary.getPhysicalPlan().explain()); ResultSchema resultSchema = new ResultSchema(summary.getLogicalPlan()); GremlinResultProcessor listener = diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/integration/ldbc/SimpleMatchTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/integration/ldbc/SimpleMatchTest.java index ea20d0823c09..49b42038d1b6 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/integration/ldbc/SimpleMatchTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/integration/ldbc/SimpleMatchTest.java @@ -133,8 +133,6 @@ public void run_simple_match_13_test() { @Test public void run_simple_match_14_test() { - // TODO: fix this in hiactor. - assumeTrue("pegasus".equals(System.getenv("ENGINE_TYPE"))); QueryContext testQuery = SimpleMatchQueries.get_simple_match_query_14_test(); Result result = session.run(testQuery.getQuery()); Assert.assertEquals(testQuery.getExpectedResult().toString(), result.list().toString()); diff --git a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/integration/movie/MovieTest.java b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/integration/movie/MovieTest.java index c85d84dd09a1..d909d47d9886 100644 --- a/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/integration/movie/MovieTest.java +++ b/interactive_engine/compiler/src/test/java/com/alibaba/graphscope/cypher/integration/movie/MovieTest.java @@ -126,7 +126,6 @@ public void run_movie_query15_test() { @Test public void run_movie_query16_test() { - assumeTrue("pegasus".equals(System.getenv("ENGINE_TYPE"))); QueryContext testQuery = MovieQueries.get_movie_query16_test(); Result result = session.run(testQuery.getQuery()); Assert.assertEquals(testQuery.getExpectedResult().toString(), result.list().toString()); @@ -134,7 +133,6 @@ public void run_movie_query16_test() { @Test public void run_movie_query17_test() { - assumeTrue("pegasus".equals(System.getenv("ENGINE_TYPE"))); QueryContext testQuery = MovieQueries.get_movie_query17_test(); Result result = session.run(testQuery.getQuery()); Assert.assertEquals(testQuery.getExpectedResult().toString(), result.list().toString()); @@ -142,7 +140,6 @@ public void run_movie_query17_test() { @Test public void run_movie_query18_test() { - assumeTrue("pegasus".equals(System.getenv("ENGINE_TYPE"))); QueryContext testQuery = MovieQueries.get_movie_query18_test(); Result result = session.run(testQuery.getQuery()); Assert.assertEquals(testQuery.getExpectedResult().toString(), result.list().toString()); @@ -150,7 +147,6 @@ public void run_movie_query18_test() { @Test public void run_movie_query19_test() { - assumeTrue("pegasus".equals(System.getenv("ENGINE_TYPE"))); QueryContext testQuery = MovieQueries.get_movie_query19_test(); Result result = session.run(testQuery.getQuery()); Assert.assertEquals(testQuery.getExpectedResult().toString(), result.list().toString()); diff --git a/interactive_engine/data-load-tool/pom.xml b/interactive_engine/data-load-tool/pom.xml index 1fb46cff251c..599fee7d947e 100644 --- a/interactive_engine/data-load-tool/pom.xml +++ b/interactive_engine/data-load-tool/pom.xml @@ -90,22 +90,6 @@ com.aliyun.oss aliyun-sdk-oss - - - org.apache.spark - spark-core_${scala.binary.version} - provided - - - org.apache.spark - spark-sql_${scala.binary.version} - provided - - - org.scala-lang - scala-library - provided - @@ -113,17 +97,6 @@ org.apache.maven.plugins maven-compiler-plugin - - net.alchim31.maven - scala-maven-plugin - - - - compile - - - - org.apache.maven.plugins maven-shade-plugin diff --git a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/DataCommand.java b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/DataCommand.java index 613b8088a111..9dd4ffa47926 100644 --- a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/DataCommand.java +++ b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/DataCommand.java @@ -1,6 +1,7 @@ package com.alibaba.graphscope.groot.dataload; import com.alibaba.graphscope.groot.common.config.DataLoadConfig; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.groot.common.schema.api.GraphSchema; import com.alibaba.graphscope.groot.common.schema.mapper.GraphSchemaMapper; import com.alibaba.graphscope.groot.dataload.databuild.ColumnMappingInfo; @@ -54,7 +55,7 @@ private void initialize() throws IOException { } dataRootPath = outputPath; } else if (dataSinkType.equalsIgnoreCase("VOLUME")) { - throw new IOException( + throw new InvalidArgumentException( "Volume only supports load.after.build mode, which is running build, ingest and" + " commit at the same driver."); } else if (dataSinkType.equalsIgnoreCase("OSS")) { @@ -64,7 +65,7 @@ private void initialize() throws IOException { ingestConfig = fs.getConfig(); } } else { - throw new IOException("Unsupported data sink: " + dataSinkType); + throw new InvalidArgumentException("Unsupported data sink: " + dataSinkType); } ObjectMapper objectMapper = new ObjectMapper(); diff --git a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/DataBuildMapper.java b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/DataBuildMapper.java index 0a7506bab3b0..5924e972588a 100644 --- a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/DataBuildMapper.java +++ b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/DataBuildMapper.java @@ -14,6 +14,7 @@ package com.alibaba.graphscope.groot.dataload.databuild; import com.alibaba.graphscope.groot.common.config.DataLoadConfig; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.groot.common.schema.api.*; import com.alibaba.graphscope.groot.common.schema.mapper.GraphSchemaMapper; import com.alibaba.graphscope.groot.common.schema.wrapper.PropertyValue; @@ -101,7 +102,7 @@ protected void map(LongWritable key, Text value, Context context) this.outKey.set(in.getArray(), in.getOffset(), in.getLength()); context.write(this.outKey, this.outVal); } else { - throw new IllegalArgumentException("Invalid label " + labelId); + throw new InvalidArgumentException("Invalid label " + labelId); } } } diff --git a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/DataBuildMapperOdps.java b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/DataBuildMapperOdps.java index 41352370704a..2b5f727053fc 100644 --- a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/DataBuildMapperOdps.java +++ b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/DataBuildMapperOdps.java @@ -16,6 +16,7 @@ package com.alibaba.graphscope.groot.dataload.databuild; import com.alibaba.graphscope.groot.common.config.DataLoadConfig; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.groot.common.schema.api.*; import com.alibaba.graphscope.groot.common.schema.mapper.GraphSchemaMapper; import com.alibaba.graphscope.groot.common.schema.wrapper.PropertyValue; @@ -99,7 +100,7 @@ public void map(long recordNum, Record record, TaskContext context) throws IOExc outKey.set(new Object[] {new String(in.getBytes(), charSet)}); context.write(outKey, outVal); } else { - throw new IllegalArgumentException("Invalid label " + labelId); + throw new InvalidArgumentException("Invalid label " + labelId); } } } diff --git a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/DataBuildMapperOdpsDebug.java b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/DataBuildMapperOdpsDebug.java index ba2b9a42577f..63e8cfe118cc 100644 --- a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/DataBuildMapperOdpsDebug.java +++ b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/DataBuildMapperOdpsDebug.java @@ -16,6 +16,7 @@ package com.alibaba.graphscope.groot.dataload.databuild; import com.alibaba.graphscope.groot.common.config.DataLoadConfig; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.groot.common.schema.api.*; import com.alibaba.graphscope.groot.common.schema.mapper.GraphSchemaMapper; import com.alibaba.graphscope.groot.common.schema.wrapper.PropertyValue; @@ -94,7 +95,7 @@ public void map(long recordNum, Record record, TaskContext context) throws IOExc outKey.set(0, getEdgeKeyEncoded(in)); context.write(outKey); } else { - throw new IllegalArgumentException("Invalid label " + labelId); + throw new InvalidArgumentException("Invalid label " + labelId); } } @@ -111,7 +112,7 @@ private String getVertexRawKeys(GraphVertex type, Map colMap, return concatenateItemsByIndices(items, indices); } - private String getEdgeRawKeys(ColumnMappingInfo info, String[] items) throws IOException { + private String getEdgeRawKeys(ColumnMappingInfo info, String[] items) { Map srcPkColMap = info.getSrcPkColMap(); Map dstPkColMap = info.getDstPkColMap(); @@ -120,11 +121,10 @@ private String getEdgeRawKeys(ColumnMappingInfo info, String[] items) throws IOE return concatenateItemsByIndices(items, pkIds); } - private static String concatenateItemsByIndices(String[] array, List indices) - throws IOException { + private static String concatenateItemsByIndices(String[] array, List indices) { StringBuilder builder = new StringBuilder(); if (indices.isEmpty()) { - throw new IOException("indices are empty!"); + throw new InvalidArgumentException("indices are empty!"); } for (int index : indices) { builder.append(array[index]).append(","); diff --git a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/DataBuildPartitionerOdps.java b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/DataBuildPartitionerOdps.java index bf7c2658ea91..b4f4458ac977 100644 --- a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/DataBuildPartitionerOdps.java +++ b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/DataBuildPartitionerOdps.java @@ -15,6 +15,7 @@ */ package com.alibaba.graphscope.groot.dataload.databuild; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.groot.common.util.PartitionUtils; import com.aliyun.odps.data.Record; import com.aliyun.odps.mapred.Partitioner; @@ -35,7 +36,7 @@ public int getPartition(Record key, Record value, int numPartitions) { try { keyBytes = ((String) key.get(0)).getBytes(DataBuildMapperOdps.charSet); } catch (UnsupportedEncodingException e) { - throw new RuntimeException("PartitionerOdps: Should not have happened " + e); + throw new InvalidArgumentException("PartitionerOdps: Should not have happened " + e); } ByteBuffer keyBuf = ByteBuffer.wrap(keyBytes); long partitionKey = keyBuf.getLong(8); diff --git a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/DataBuildReducerOdps.java b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/DataBuildReducerOdps.java index c32a8c53b2e0..addb84fabd28 100644 --- a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/DataBuildReducerOdps.java +++ b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/DataBuildReducerOdps.java @@ -16,6 +16,7 @@ package com.alibaba.graphscope.groot.dataload.databuild; import com.alibaba.graphscope.groot.common.config.DataLoadConfig; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.groot.dataload.util.AbstractFileSystem; import com.alibaba.graphscope.groot.dataload.util.FSFactory; import com.aliyun.odps.data.Record; @@ -130,7 +131,7 @@ public String getFileMD5(String fileName) throws IOException { } return new String(Hex.encodeHex(MD5.digest())); } catch (NoSuchAlgorithmException | IOException e) { - throw new IOException(e); + throw new InvalidArgumentException(e); } finally { if (fis != null) { fis.close(); diff --git a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/FileColumnMapping.java b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/FileColumnMapping.java index 05249d3d93c0..15a9c3ff6681 100644 --- a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/FileColumnMapping.java +++ b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/FileColumnMapping.java @@ -15,7 +15,7 @@ */ package com.alibaba.graphscope.groot.dataload.databuild; -import com.alibaba.graphscope.groot.common.exception.InvalidSchemaException; +import com.alibaba.graphscope.groot.common.exception.IllegalSchemaException; import com.alibaba.graphscope.groot.common.schema.api.*; import java.util.HashMap; @@ -104,7 +104,7 @@ public ColumnMappingInfo toColumnMappingInfo(GraphSchema graphSchema) { propertiesMap); } } - throw new InvalidSchemaException( + throw new IllegalSchemaException( "invalid mapping for label [" + this.label + "] srcLabel [" diff --git a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/OfflineBuildOdps.java b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/OfflineBuildOdps.java index 252873f8bfff..7271d57f12b5 100644 --- a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/OfflineBuildOdps.java +++ b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/OfflineBuildOdps.java @@ -14,6 +14,7 @@ package com.alibaba.graphscope.groot.dataload.databuild; import com.alibaba.graphscope.groot.common.config.DataLoadConfig; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.groot.common.schema.api.GraphSchema; import com.alibaba.graphscope.groot.common.schema.mapper.GraphSchemaMapper; import com.alibaba.graphscope.groot.common.schema.wrapper.GraphDef; @@ -37,7 +38,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.util.*; public class OfflineBuildOdps { @@ -161,9 +161,9 @@ public static void main(String[] args) throws Exception { String outputTable = properties.getProperty(DataLoadConfig.OUTPUT_TABLE); OutputUtils.addTable(Utils.parseTableURL(odps, outputTable), job); } else if (dataSinkType.equalsIgnoreCase("HDFS")) { - throw new IOException("HDFS as a data sink is not supported in ODPS"); + throw new InvalidArgumentException("HDFS as a data sink is not supported in ODPS"); } else { - throw new IOException("Unsupported data sink: " + dataSinkType); + throw new InvalidArgumentException("Unsupported data sink: " + dataSinkType); } String schemaJson = GraphSchemaMapper.parseFromSchema(schema).toJsonString(); @@ -183,7 +183,7 @@ public static void main(String[] args) throws Exception { try { JobClient.runJob(job); } catch (Exception e) { - throw new IOException(e); + throw new InvalidArgumentException(e); } String _tmp = properties.getProperty(DataLoadConfig.LOAD_AFTER_BUILD, "true"); diff --git a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/SstOutputFormat.java b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/SstOutputFormat.java index 34b0baccd02c..2507b2c31d14 100644 --- a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/SstOutputFormat.java +++ b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/SstOutputFormat.java @@ -15,6 +15,8 @@ */ package com.alibaba.graphscope.groot.dataload.databuild; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -64,7 +66,7 @@ public void write(BytesWritable key, BytesWritable value) throws IOException { ByteBuffer buffer = ByteBuffer.wrap(key.copyBytes()); long tableId = buffer.getLong(0) >> 1; long hashId = buffer.getLong(8); - throw new IOException( + throw new InvalidArgumentException( "Write SST Error! TableId: [" + tableId + "], hashId: [" + hashId + "]", e); } } @@ -74,7 +76,7 @@ public void close(TaskAttemptContext context) throws IOException { try { sstFileWriter.finish(); } catch (RocksDBException e) { - throw new IOException(e); + throw new InvalidArgumentException(e); } fs.copyFromLocalFile(true, new Path(fileName), path); } @@ -91,7 +93,7 @@ public RecordWriter getRecordWriter(TaskAttemptCon try { return new SstRecordWriter(fs, file); } catch (RocksDBException e) { - throw new IOException(e); + throw new InvalidArgumentException(e); } } } diff --git a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/Utils.java b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/Utils.java index dc85b2b2ddf8..75715a9f7517 100644 --- a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/Utils.java +++ b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/databuild/Utils.java @@ -1,6 +1,7 @@ package com.alibaba.graphscope.groot.dataload.databuild; -import com.alibaba.graphscope.groot.common.exception.PropertyDefNotFoundException; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; +import com.alibaba.graphscope.groot.common.exception.NotFoundException; import com.alibaba.graphscope.groot.common.schema.api.*; import com.alibaba.graphscope.groot.common.schema.wrapper.PropertyValue; import com.alibaba.graphscope.groot.dataload.unified.*; @@ -201,12 +202,12 @@ public static Map buildProperties( String msg = "Label [" + label + "]: "; if (prop == null) { msg += "propertyId [" + propertyId + "] not found"; - throw new PropertyDefNotFoundException(msg); + throw new NotFoundException(msg); } if (colIdx >= items.length) { msg += "Invalid mapping [" + colIdx + "]->[" + propertyId + "]"; msg += "Data: " + Arrays.toString(items); - throw new IllegalArgumentException(msg); + throw new InvalidArgumentException(msg); } PropertyValue propertyValue = null; if (items[colIdx] != null) { @@ -266,7 +267,7 @@ public static String convertDateForLDBC(String input) { try { return DST_FMT.format(SRC_FMT.parse(input)); } catch (ParseException e) { - throw new RuntimeException(e); + throw new InvalidArgumentException(e); } } diff --git a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/util/FSFactory.java b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/util/FSFactory.java index fb360e0e7e1c..5f008e09fd3c 100644 --- a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/util/FSFactory.java +++ b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/util/FSFactory.java @@ -1,6 +1,7 @@ package com.alibaba.graphscope.groot.dataload.util; import com.alibaba.graphscope.groot.common.config.DataLoadConfig; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.aliyun.odps.mapred.conf.JobConf; import java.io.IOException; @@ -15,9 +16,9 @@ public static AbstractFileSystem Create(JobConf conf) throws IOException { } else if (dataSinkType.equalsIgnoreCase("OSS")) { return new OSSFS(conf); } else if (dataSinkType.equalsIgnoreCase("HDFS")) { - throw new IOException("HDFS as a data sink is not supported in ODPS"); + throw new InvalidArgumentException("HDFS as a data sink is not supported in ODPS"); } else { - throw new IOException("Unsupported data sink: " + dataSinkType); + throw new InvalidArgumentException("Unsupported data sink: " + dataSinkType); } } } diff --git a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/util/HttpClient.java b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/util/HttpClient.java index d984f4a392ec..b7d85486b481 100644 --- a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/util/HttpClient.java +++ b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/util/HttpClient.java @@ -1,5 +1,7 @@ package com.alibaba.graphscope.groot.dataload.util; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; + import org.apache.commons.codec.binary.Base64; import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; @@ -81,14 +83,14 @@ public static String doGet(String url, Map param) throws Excepti if (response.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { resultMsg = EntityUtils.toString(response.getEntity(), "UTF-8"); } else { - throw new Exception( - "่ฐƒ็”จhttp-getๆ–นๆณ•่ฟ”ๅ›žๅคฑ่ดฅcode=" + throw new InvalidArgumentException( + "HTTP GET failed. code=" + response.getStatusLine().getStatusCode() - + ",msg=" + + ", msg=" + response.getStatusLine().getReasonPhrase()); } } catch (Exception e) { - throw new Exception("่ฐƒ็”จhttp-getๆ–นๆณ•ๅผ‚ๅธธ", e); + throw new InvalidArgumentException("HTTP GET failed", e); } finally { try { if (response != null) { @@ -96,7 +98,7 @@ public static String doGet(String url, Map param) throws Excepti } httpClient.close(); } catch (IOException e) { - throw new Exception("httpClient closeๅผ‚ๅธธ", e); + throw new InvalidArgumentException("httpClient close failed", e); } } return resultMsg; diff --git a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/util/OSSFS.java b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/util/OSSFS.java index fbfc378662cf..89bcc3ebbda9 100644 --- a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/util/OSSFS.java +++ b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/util/OSSFS.java @@ -1,6 +1,7 @@ package com.alibaba.graphscope.groot.dataload.util; import com.alibaba.graphscope.groot.common.config.DataLoadConfig; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.groot.dataload.unified.UniConfig; import com.aliyun.odps.mapred.TaskContext; import com.aliyun.odps.mapred.conf.JobConf; @@ -40,7 +41,7 @@ private void initClient() throws IOException { this.ossClient = new OSSClientBuilder().build(endpoint, ossAccessID, ossAccessKey, conf); } catch (OSSException | ClientException oe) { - throw new IOException(oe); + throw new InvalidArgumentException(oe); } } @@ -77,7 +78,7 @@ public OSSFS(UniConfig properties) throws IOException { } @Override - public void open(TaskContext context, String mode) throws IOException {} + public void open(TaskContext context, String mode) {} public String getQualifiedPath() { return "oss://"; @@ -123,7 +124,7 @@ public void putObject(String prefix, String fileName) throws IOException { } } - public void uploadFileWithCheckPoint(String srcFile, String dstFile) throws IOException { + public void uploadFileWithCheckPoint(String srcFile, String dstFile) { String key = Paths.get(object, dstFile).toString(); UploadFileRequest uploadFileRequest = new UploadFileRequest(bucket, key); uploadFileRequest.setUploadFile(srcFile); @@ -133,10 +134,10 @@ public void uploadFileWithCheckPoint(String srcFile, String dstFile) throws IOEx ossClient.uploadFile(uploadFileRequest); } catch (OSSException oe) { logger.error("Error message: {}", oe.getMessage()); - throw new IOException(oe); + throw new InvalidArgumentException(oe); } catch (Throwable ce) { logger.error("Error Message:" + ce.getMessage()); - throw new IOException(ce); + throw new InvalidArgumentException(ce); } } @@ -169,7 +170,7 @@ public String readToString(String fileName) throws IOException { ossObject.close(); } catch (OSSException | ClientException oe) { logger.error("Error Message:" + oe.getMessage()); - throw new IOException(oe); + throw new InvalidArgumentException(oe); } return data.toString(); } diff --git a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/util/VolumeFS.java b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/util/VolumeFS.java index 238f6a92812f..431edd62f990 100644 --- a/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/util/VolumeFS.java +++ b/interactive_engine/data-load-tool/src/main/java/com/alibaba/graphscope/groot/dataload/util/VolumeFS.java @@ -1,6 +1,7 @@ package com.alibaba.graphscope.groot.dataload.util; import com.alibaba.graphscope.groot.common.config.DataLoadConfig; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.groot.dataload.unified.UniConfig; import com.aliyun.odps.Odps; import com.aliyun.odps.OdpsException; @@ -80,7 +81,7 @@ public Map setConfig(Odps odps) throws IOException { AliyunAccount aliyunAccount = account instanceof AliyunAccount ? ((AliyunAccount) account) : null; if (aliyunAccount == null) { - throw new IOException("Not an AliyunAccount"); + throw new InvalidArgumentException("Not an AliyunAccount"); } HashMap config = new HashMap<>(); config.put(DataLoadConfig.ODPS_ACCESS_ID, aliyunAccount.getAccessId()); @@ -175,7 +176,7 @@ public void createVolumeIfNotExists(Odps odps) throws IOException { + e.getErrorCode() + "]: " + e.getMessage()); - throw new IOException(e.getMessage()); + throw new InvalidArgumentException(e.getMessage()); } } diff --git a/interactive_engine/data-load-tool/src/main/scala/com/alibaba/graphscope/groot/dataload/LoadToolSpark.scala b/interactive_engine/data-load-tool/src/main/scala/com/alibaba/graphscope/groot/dataload/LoadToolSpark.scala deleted file mode 100644 index cbcdfa4aa0f4..000000000000 --- a/interactive_engine/data-load-tool/src/main/scala/com/alibaba/graphscope/groot/dataload/LoadToolSpark.scala +++ /dev/null @@ -1,45 +0,0 @@ -/** - * Copyright 2020 Alibaba Group Holding Limited. - * - *

Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file - * except in compliance with the License. You may obtain a copy of the License at - * - *

http://www.apache.org/licenses/LICENSE-2.0 - * - *

Unless required by applicable law or agreed to in writing, software distributed under the - * License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.alibaba.graphscope.groot.dataload - -import org.apache.spark.sql.SparkSession - -object LoadToolSpark { - - def main(args: Array[String]) { - - val spark = SparkSession - .builder() - .appName("LoadToolSpark") - .getOrCreate() - - try { - val command = args(0) - val configPath = args(1) - if ("ingest".equalsIgnoreCase(command)) { - LoadTool.ingest(configPath) - } else if ("commit".equalsIgnoreCase(command)) { - LoadTool.commit(configPath) - } else if ("ingestAndCommit".equalsIgnoreCase(command)) { - LoadTool.ingest(configPath) - LoadTool.commit(configPath) - } else { - throw new Exception("supported COMMAND: ingest / commit / ingestAndCommit"); - } - } catch { - case e: Throwable => throw e - } - spark.close() - } -} diff --git a/interactive_engine/executor/assembly/groot/src/store/jna_response.rs b/interactive_engine/executor/assembly/groot/src/store/jna_response.rs index 405cfda77230..952d362ee9a0 100644 --- a/interactive_engine/executor/assembly/groot/src/store/jna_response.rs +++ b/interactive_engine/executor/assembly/groot/src/store/jna_response.rs @@ -3,7 +3,7 @@ use std::fmt; use std::fmt::Formatter; use std::os::raw::{c_char, c_void}; -use groot_store::db::api::{GraphError, GraphErrorCode, GraphResult}; +use groot_store::db::api::{ErrorCode, GraphError, GraphResult}; #[repr(C)] #[allow(non_snake_case)] @@ -52,7 +52,7 @@ impl JnaResponse { pub fn data(&mut self, data: Vec) -> GraphResult<()> { if data.len() != data.capacity() { let msg = format!("data len {} must eq capacity {}", data.len(), data.capacity()); - return Err(GraphError::new(GraphErrorCode::InvalidData, msg)); + return Err(GraphError::new(ErrorCode::INVALID_DATA, msg)); } self.data = data.as_ptr() as *const c_void; self.len = data.len() as i32; diff --git a/interactive_engine/executor/engine/pegasus/network/src/receive/mod.rs b/interactive_engine/executor/engine/pegasus/network/src/receive/mod.rs index ba8c651a83e0..3307391635e2 100644 --- a/interactive_engine/executor/engine/pegasus/network/src/receive/mod.rs +++ b/interactive_engine/executor/engine/pegasus/network/src/receive/mod.rs @@ -82,10 +82,15 @@ fn add_remote_register(local: u64, remote: u64, register: InboxRegister) { lock.insert((local, remote), register); } -fn remove_remote_register(local: u64, remote: u64) -> Option { +fn remove_remote_register(local: u64, remote: u64, other: InboxRegister) -> Option { let mut lock = REMOTE_RECV_REGISTER .write() .expect("failure to lock REMOTE_RECV_REGISTER"); + if let Some(register) = lock.get(&(local, remote)) { + if !register.from_same_receiver(&other) { + return None; + } + } lock.remove(&(local, remote)) } @@ -135,7 +140,7 @@ pub fn start_net_receiver( break; } } - remove_remote_register(local, remote.id); + remove_remote_register(local, remote.id, net_recv.get_inbox_register()); info!("IPC receiver recv from {:?} exit;", remote); }) .expect("start net recv thread failure;"); diff --git a/interactive_engine/executor/engine/pegasus/network/src/receive/net_rx.rs b/interactive_engine/executor/engine/pegasus/network/src/receive/net_rx.rs index a49794f45664..844e562e3954 100644 --- a/interactive_engine/executor/engine/pegasus/network/src/receive/net_rx.rs +++ b/interactive_engine/executor/engine/pegasus/network/src/receive/net_rx.rs @@ -278,6 +278,10 @@ impl InboxRegister { pub(crate) fn register(&self, channel_id: u128, tx: &MessageSender) -> Result<(), NetError> { self.inner.register(channel_id, tx.clone()) } + + pub(crate) fn from_same_receiver(&self, other: &InboxRegister) -> bool { + Arc::ptr_eq(&self.inner, &other.inner) + } } #[cfg(test)] diff --git a/interactive_engine/executor/engine/pegasus/network/src/send/mod.rs b/interactive_engine/executor/engine/pegasus/network/src/send/mod.rs index fd7e58dad6d6..f6d3fecceba5 100644 --- a/interactive_engine/executor/engine/pegasus/network/src/send/mod.rs +++ b/interactive_engine/executor/engine/pegasus/network/src/send/mod.rs @@ -159,10 +159,16 @@ pub(crate) fn add_remote_sender(local_id: u64, server: &Server, tx: &Arc>) { let mut lock = REMOTE_MSG_SENDER .write() .expect("REMOTE_MSG_SENDER write lock poisoned"); + if let Some((_, tx)) = lock.get(&(local_id, remote_id)) { + let weak = Arc::downgrade(other); + if !Weak::ptr_eq(tx, &weak) { + return; + } + } lock.remove(&(local_id, remote_id)); } @@ -275,5 +281,5 @@ fn busy_send( } } info!("IPC sender to {:?} exit;", remote); - remove_remote_sender(local, remote); + remove_remote_sender(local, remote, net_tx.get_outbox_tx().as_ref().expect("")); } diff --git a/interactive_engine/executor/store/groot/build.rs b/interactive_engine/executor/store/groot/build.rs index 1c92ad4c1efe..fe123a94ae35 100644 --- a/interactive_engine/executor/store/groot/build.rs +++ b/interactive_engine/executor/store/groot/build.rs @@ -7,6 +7,7 @@ fn main() { proto_root.to_owned() + "/groot/sdk/model.proto", proto_root.to_owned() + "/groot/sdk/schema.proto", proto_root.to_owned() + "/schema_common.proto", + proto_root.to_owned() + "/error/insight.proto", ], &[proto_root], "./src/db/proto", diff --git a/interactive_engine/executor/store/groot/src/db/api/error.rs b/interactive_engine/executor/store/groot/src/db/api/error.rs index 5c79dc3e683d..bd1b030dee9d 100644 --- a/interactive_engine/executor/store/groot/src/db/api/error.rs +++ b/interactive_engine/executor/store/groot/src/db/api/error.rs @@ -1,118 +1,43 @@ use std::fmt::Debug; +use protobuf::ProtobufEnum; + +pub use crate::db::proto::insight::Code as ErrorCode; + #[derive(Clone)] -pub struct ErrorWithBackTrace { - err_code: GraphErrorCode, +pub struct GraphError { + err_code: ErrorCode, + ec: String, msg: String, backtrace: Vec<(String, String)>, } -impl ErrorWithBackTrace { - pub fn new(err_code: GraphErrorCode, msg: String) -> Self { - ErrorWithBackTrace { err_code, msg, backtrace: Vec::new() } +impl GraphError { + pub fn new(err_code: ErrorCode, msg: String) -> Self { + let ec = format!("06-{:04}", err_code.value()); + GraphError { err_code, ec, msg, backtrace: Vec::new() } } pub fn add_backtrace(&mut self, function: String, code_info: String) { self.backtrace.push((function, code_info)); } - - pub fn get_error_code(&self) -> GraphErrorCode { + pub fn get_error_code(&self) -> ErrorCode { self.err_code } } -#[derive(Clone)] -pub enum GraphError { - WithBackTrace(ErrorWithBackTrace), - Internal(String), - Rocksdb(String), - InvalidArgument(String), - TooManyVersions(usize), -} - -impl GraphError { - pub fn new(err_code: GraphErrorCode, msg: String) -> Self { - let inner = ErrorWithBackTrace { err_code, msg, backtrace: Vec::new() }; - GraphError::WithBackTrace(inner) - } - - pub fn add_backtrace(&mut self, function: String, code_info: String) { - match self { - GraphError::WithBackTrace(inner) => { - inner.add_backtrace(function, code_info); - } - _ => { - unimplemented!() - } - } - } - - pub fn get_error_code(&self) -> GraphErrorCode { - match self { - GraphError::WithBackTrace(inner) => inner.get_error_code(), - _ => { - unimplemented!() - } - } - } -} - impl Debug for GraphError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> Result<(), std::fmt::Error> { - match self { - GraphError::WithBackTrace(inner) => { - writeln!(f, "")?; - writeln!(f, "error code: {:?}, msg: {}", inner.err_code, inner.msg)?; - for bt in inner.backtrace.iter().rev() { - writeln!(f, "\t{}", bt.0)?; - writeln!(f, "\t\tat {}", bt.1)?; - } - write!(f, "") - } - GraphError::Internal(s) | GraphError::Rocksdb(s) | GraphError::InvalidArgument(s) => { - write!(f, "{}", s) - } - GraphError::TooManyVersions(limit) => write!(f, "version count exceed limit {}", limit), + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + writeln!(f, "")?; + writeln!(f, "error code: {:?}, ec: {}, msg: {}", self.err_code, self.ec, self.msg)?; + for bt in self.backtrace.iter().rev() { + writeln!(f, "\t{}", bt.0)?; + writeln!(f, "\t\tat {}", bt.1)?; } + write!(f, "") } } -#[allow(dead_code)] -#[derive(Debug, Copy, Clone)] -pub enum GraphErrorCode { - // try to get a property in ValueType1, but the it's real type is ValueType2 and it's not ValueType1 and - // compatible to ValueType1 - ValueTypeMismatch, - // transform bytes to str failed because it's not in utf-8 format - Utf8Error, - // something error with binary data in storage - InvalidData, - // get lock failed - LockFailed, - // too many data of old versions in graph store, maybe something error with garbage collection - TooManyVersions, - // some fatal bug in graph store - GraphStoreBug, - // user's operation is invalid, like: create a type that already exists - InvalidOperation, - // when try to insert data, the type exists in storage but isn't visible at that moment - DataNotExists, - // vertex type or edge type is not found - TypeNotFound, - // vertex type or edge type already exists - TypeAlreadyExists, - // error in external storage like rocksdb - ExternalStorageError, - // decode property from bytes failed - DecodeError, - // vertex type not found, edge not found - MetaNotFound, - // operations or features is not supported - NotSupported, - // engine error - EngineError, -} - macro_rules! func_signature { ($func:tt, $($x:tt),*) => { { diff --git a/interactive_engine/executor/store/groot/src/db/api/property.rs b/interactive_engine/executor/store/groot/src/db/api/property.rs index d28cf936cf3a..b1622bb9d6d2 100644 --- a/interactive_engine/executor/store/groot/src/db/api/property.rs +++ b/interactive_engine/executor/store/groot/src/db/api/property.rs @@ -91,7 +91,7 @@ impl ValueType { x if x == ValueType::StringList as i32 => Ok(ValueType::StringList), _ => { let msg = format!("invalid input"); - let err = gen_graph_err!(GraphErrorCode::InvalidData, msg, from_i32, x); + let err = gen_graph_err!(ErrorCode::INVALID_DATA, msg, from_i32, x); Err(err) } } @@ -102,7 +102,7 @@ impl ValueType { match DataTypePb::from_i32(v) { None => { let msg = format!("invalid input"); - let err = gen_graph_err!(GraphErrorCode::InvalidData, msg, to_proto, v); + let err = gen_graph_err!(ErrorCode::INVALID_DATA, msg, to_proto, v); Err(err) } Some(pb) => Ok(pb), @@ -199,7 +199,7 @@ impl<'a> ValueRef<'a> { let res = self.check_type_match(ValueType::String); res_unwrap!(res, get_str)?; ::std::str::from_utf8(self.data) - .map_err(|e| gen_graph_err!(GraphErrorCode::Utf8Error, e.to_string(), get_str)) + .map_err(|e| gen_graph_err!(ErrorCode::INVALID_DATA, e.to_string(), get_str)) } pub fn get_bytes(&self) -> GraphResult<&[u8]> { @@ -285,7 +285,7 @@ impl<'a> ValueRef<'a> { pub fn check_type_match(&self, value_type: ValueType) -> GraphResult<()> { if self.r#type != value_type { let msg = format!("cannot transform {:?} to {:?}", self.r#type, value_type); - let err = gen_graph_err!(GraphErrorCode::ValueTypeMismatch, msg, check_type_match, value_type); + let err = gen_graph_err!(ErrorCode::VALUE_TYPE_MISMATCH, msg, check_type_match, value_type); return Err(err); } Ok(()) @@ -302,7 +302,7 @@ impl<'a> ValueRef<'a> { return Ok(reader); } let msg = format!("invalid {:?} bytes, data len is {}", value_type, self.data.len()); - let err = gen_graph_err!(GraphErrorCode::InvalidData, msg, check_numeric_array); + let err = gen_graph_err!(ErrorCode::INVALID_DATA, msg, check_numeric_array); Err(err) } @@ -318,7 +318,7 @@ impl<'a> ValueRef<'a> { return Ok(reader); } let msg = format!("invalid str array bytes"); - let err = gen_graph_err!(GraphErrorCode::InvalidData, msg, weak_check_str_list); + let err = gen_graph_err!(ErrorCode::INVALID_DATA, msg, weak_check_str_list); Err(err) } } diff --git a/interactive_engine/executor/store/groot/src/db/api/schema.rs b/interactive_engine/executor/store/groot/src/db/api/schema.rs index 42088d5a3948..7976e428e1de 100644 --- a/interactive_engine/executor/store/groot/src/db/api/schema.rs +++ b/interactive_engine/executor/store/groot/src/db/api/schema.rs @@ -51,7 +51,7 @@ impl GraphDef { pub fn add_type(&mut self, label: LabelId, type_def: TypeDef) -> GraphResult<()> { if self.label_to_types.contains_key(&label) { let msg = format!("labelId {}", label); - return Err(GraphError::new(GraphErrorCode::TypeAlreadyExists, msg)); + return Err(GraphError::new(ErrorCode::ALREADY_EXISTS, msg)); } for property in type_def.get_prop_defs() { if property.id > self.property_idx { @@ -67,7 +67,7 @@ impl GraphDef { pub fn update_type(&mut self, label: LabelId, type_def: TypeDef) -> GraphResult<()> { if !self.label_to_types.contains_key(&label) { let msg = format!("labelId {}", label); - return Err(GraphError::new(GraphErrorCode::TypeNotFound, msg)); + return Err(GraphError::new(ErrorCode::TYPE_NOT_FOUND, msg)); } for property in type_def.get_prop_defs() { if property.id > self.property_idx { @@ -250,7 +250,7 @@ impl TypeDef { Ok(b) => Ok(b), Err(e) => { let msg = format!("{:?}", e); - Err(gen_graph_err!(GraphErrorCode::InvalidData, msg, to_bytes)) + Err(gen_graph_err!(ErrorCode::INVALID_DATA, msg, to_bytes)) } } } diff --git a/interactive_engine/executor/store/groot/src/db/common/bytes/transform.rs b/interactive_engine/executor/store/groot/src/db/common/bytes/transform.rs index 276dd61a7dea..03a0f6b87ba7 100644 --- a/interactive_engine/executor/store/groot/src/db/common/bytes/transform.rs +++ b/interactive_engine/executor/store/groot/src/db/common/bytes/transform.rs @@ -27,7 +27,7 @@ pub fn bytes_to_i64(buf: &[u8]) -> GraphResult { Ok(reader.read_i64(0)) } else { let msg = format!("bytes with length {} cannot transform to i64", buf.len()); - let err = gen_graph_err!(GraphErrorCode::InvalidData, msg, bytes_to_i64); + let err = gen_graph_err!(ErrorCode::INVALID_DATA, msg, bytes_to_i64); Err(err) } } @@ -42,6 +42,6 @@ pub fn i64_to_arr(x: i64) -> [u8; 8] { pub fn bytes_to_str(buf: &[u8]) -> GraphResult<&str> { std::str::from_utf8(buf).map_err(|e| { let msg = format!("{:?}", e); - gen_graph_err!(GraphErrorCode::InvalidData, msg, bytes_to_str) + gen_graph_err!(ErrorCode::INVALID_DATA, msg, bytes_to_str) }) } diff --git a/interactive_engine/executor/store/groot/src/db/common/bytes/util.rs b/interactive_engine/executor/store/groot/src/db/common/bytes/util.rs index 78c571ee979f..3ee6aef3afb6 100644 --- a/interactive_engine/executor/store/groot/src/db/common/bytes/util.rs +++ b/interactive_engine/executor/store/groot/src/db/common/bytes/util.rs @@ -3,8 +3,7 @@ use std::marker::PhantomData; use protobuf::Message; -use crate::db::api::GraphErrorCode::InvalidData; -use crate::db::api::{GraphError, GraphResult}; +use crate::db::api::{ErrorCode, GraphError, GraphResult}; /// define the size that a length field takes in encoding an array pub const LEN_SIZE: usize = ::std::mem::size_of::(); @@ -114,7 +113,7 @@ impl UnsafeBytesWriter { } pub fn parse_pb(buf: &[u8]) -> GraphResult { - Message::parse_from_bytes(buf).map_err(|e| GraphError::new(InvalidData, format!("{:?}", e))) + Message::parse_from_bytes(buf).map_err(|e| GraphError::new(ErrorCode::INVALID_DATA, format!("{:?}", e))) } #[cfg(test)] diff --git a/interactive_engine/executor/store/groot/src/db/common/concurrency/volatile.rs b/interactive_engine/executor/store/groot/src/db/common/concurrency/volatile.rs index fc89cf430edc..071586e1edab 100644 --- a/interactive_engine/executor/store/groot/src/db/common/concurrency/volatile.rs +++ b/interactive_engine/executor/store/groot/src/db/common/concurrency/volatile.rs @@ -1,5 +1,3 @@ -use std::cell::UnsafeCell; - #[derive(Clone)] pub struct Volatile { data: T, @@ -33,6 +31,6 @@ impl Volatile { #[rustversion::since(1.76.0)] pub fn set(&self, data: T) { let ptr = &self.data as *const T; - unsafe { std::ptr::write_volatile((&*(ptr as *mut UnsafeCell)).get(), data) } + unsafe { std::ptr::write_volatile((&*(ptr as *mut std::cell::UnsafeCell)).get(), data) } } } diff --git a/interactive_engine/executor/store/groot/src/db/common/str.rs b/interactive_engine/executor/store/groot/src/db/common/str.rs index 4e958bf20dd3..d5b036139e8d 100644 --- a/interactive_engine/executor/store/groot/src/db/common/str.rs +++ b/interactive_engine/executor/store/groot/src/db/common/str.rs @@ -4,20 +4,20 @@ use crate::db::api::*; pub fn str_to_i64(s: &str) -> GraphResult { s.parse().map_err(|e| { let msg = format!("{:?}", e); - gen_graph_err!(GraphErrorCode::InvalidData, msg, str_to_i64, s) + gen_graph_err!(ErrorCode::INVALID_DATA, msg, str_to_i64, s) }) } pub fn str_to_i32(s: &str) -> GraphResult { s.parse().map_err(|e| { let msg = format!("{:?}", e); - gen_graph_err!(GraphErrorCode::InvalidData, msg, str_to_i64, s) + gen_graph_err!(ErrorCode::INVALID_DATA, msg, str_to_i64, s) }) } pub fn parse_str(s: &str) -> GraphResult { s.parse().map_err(|_| { let msg = format!("parse str failed"); - gen_graph_err!(GraphErrorCode::InvalidData, msg, parse_str, s) + gen_graph_err!(ErrorCode::INVALID_DATA, msg, parse_str, s) }) } diff --git a/interactive_engine/executor/store/groot/src/db/graph/bin.rs b/interactive_engine/executor/store/groot/src/db/graph/bin.rs index bca2941fcc33..d7f66da9557d 100644 --- a/interactive_engine/executor/store/groot/src/db/graph/bin.rs +++ b/interactive_engine/executor/store/groot/src/db/graph/bin.rs @@ -17,7 +17,7 @@ pub fn vertex_key(table_id: TableId, id: VertexId, ts: SnapshotId) -> [u8; 24] { pub fn parse_vertex_key(key: &[u8]) -> GraphResult<(VertexId, SnapshotId)> { if key.len() != 24 { let msg = format!("invalid key, key len is {}", key.len()); - let err = gen_graph_err!(GraphErrorCode::InvalidData, msg, parse_vertex_key); + let err = gen_graph_err!(ErrorCode::INVALID_DATA, msg, parse_vertex_key); error!("parse_vertex_key failed. error: {:?}", err); return Err(err); } diff --git a/interactive_engine/executor/store/groot/src/db/graph/codec.rs b/interactive_engine/executor/store/groot/src/db/graph/codec.rs index b8ab06d6a343..8526a99e9c4e 100644 --- a/interactive_engine/executor/store/groot/src/db/graph/codec.rs +++ b/interactive_engine/executor/store/groot/src/db/graph/codec.rs @@ -402,7 +402,7 @@ impl Encoder { // if let Some(data) = props.get(info.prop_id) { // if !check_fixed_prop_len(info.r#type, data) { // let msg = format!("invalid {:?} data", info.r#type); - // let err = gen_graph_err!(GraphErrorCode::InvalidData, msg); + // let err = gen_graph_err!(GraphErrorCode::INVALID_DATA, msg); // return Err(err); // } // } @@ -498,7 +498,7 @@ fn check_var_len_prop(r#type: ValueType, data: &[u8]) -> GraphResult<()> { return Ok(()); } let msg = format!("string is not utf-8"); - let err = gen_graph_err!(GraphErrorCode::InvalidData, msg); + let err = gen_graph_err!(ErrorCode::INVALID_DATA, msg); return Err(err); } ValueType::Bytes => { @@ -523,7 +523,7 @@ fn check_var_len_prop(r#type: ValueType, data: &[u8]) -> GraphResult<()> { _ => unreachable!(), } let msg = format!("invalid {:?} data, length is not right", r#type); - let err = gen_graph_err!(GraphErrorCode::InvalidData, msg); + let err = gen_graph_err!(ErrorCode::INVALID_DATA, msg); Err(err) } @@ -548,13 +548,13 @@ impl CodecManager { let mut max_version = res_unwrap!(self.lock.lock(), add_codec, si, codec)?; if self.versions.get_latest_version() >= si { let msg = format!("si#{} is too small, cannot add", si); - let err = gen_graph_err!(GraphErrorCode::InvalidOperation, msg, add_codec, si, codec); + let err = gen_graph_err!(ErrorCode::INVALID_OPERATION, msg, add_codec, si, codec); return Err(err); } let version = codec.get_version(); if version <= *max_version { let msg = format!("current max codec version is {}, you cannot add a older one", *max_version); - let err = gen_graph_err!(GraphErrorCode::InvalidOperation, msg, add_codec, si, codec); + let err = gen_graph_err!(ErrorCode::INVALID_OPERATION, msg, add_codec, si, codec); return Err(err); } let codec = Arc::new(codec); @@ -563,7 +563,7 @@ impl CodecManager { let map_ref = unsafe { map.as_ref() }.ok_or_else(|| { let msg = "get map reference return `None`".to_string(); - gen_graph_err!(GraphErrorCode::InvalidData, msg, get_map, si) + gen_graph_err!(ErrorCode::INVALID_DATA, msg, get_map, si) })?; let mut map_clone = map_ref.clone(); map_clone.insert(version, codec); @@ -582,12 +582,12 @@ impl CodecManager { let map = self.get_map(&guard); let map_ref = unsafe { map.as_ref() }.ok_or_else(|| { let msg = "get map reference return `None`".to_string(); - gen_graph_err!(GraphErrorCode::InvalidData, msg, get_map, si) + gen_graph_err!(ErrorCode::INVALID_DATA, msg, get_map, si) })?; return get_codec(map_ref, version).map(|codec| Encoder::new(codec)); } let msg = format!("codec not found at si#{}", si); - let err = gen_graph_err!(GraphErrorCode::MetaNotFound, msg, get_encoder, si); + let err = gen_graph_err!(ErrorCode::NOT_FOUND, msg, get_encoder, si); Err(err) } @@ -598,14 +598,14 @@ impl CodecManager { let map = self.get_map(&guard); let map_ref = unsafe { map.as_ref() }.ok_or_else(|| { let msg = "get map reference return `None`".to_string(); - gen_graph_err!(GraphErrorCode::InvalidData, msg, get_map, si, version) + gen_graph_err!(ErrorCode::INVALID_DATA, msg, get_map, si, version) })?; let src = res_unwrap!(get_codec(map_ref, version), get_decoder, si, version)?; let target = res_unwrap!(get_codec(map_ref, target_version), get_decoder, si, version)?; return Ok(Decoder::new(target, src)); } let msg = format!("codec not found at si#{}", si); - let err = gen_graph_err!(GraphErrorCode::MetaNotFound, msg, get_decoder, si, version); + let err = gen_graph_err!(ErrorCode::NOT_FOUND, msg, get_decoder, si, version); Err(err) } @@ -616,7 +616,7 @@ impl CodecManager { let map = self.get_map(&guard); let map_ref = unsafe { map.as_ref() }.ok_or_else(|| { let msg = "get map reference return `None`".to_string(); - gen_graph_err!(GraphErrorCode::InvalidData, msg, get_map, version) + gen_graph_err!(ErrorCode::INVALID_DATA, msg, get_map, version) })?; let mut map_clone = map_ref.clone(); if map_clone.remove(&version).is_some() { @@ -639,7 +639,7 @@ impl CodecManager { fn get_codec(map: &CodecMap, version: CodecVersion) -> GraphResult> { map.get(&version).cloned().ok_or_else(|| { let msg = format!("codec of version#{} not found", version); - gen_graph_err!(GraphErrorCode::MetaNotFound, msg, get_encoder, version) + gen_graph_err!(ErrorCode::NOT_FOUND, msg, get_encoder, version) }) } diff --git a/interactive_engine/executor/store/groot/src/db/graph/meta.rs b/interactive_engine/executor/store/groot/src/db/graph/meta.rs index 8efd4e77b085..6f184d123b24 100644 --- a/interactive_engine/executor/store/groot/src/db/graph/meta.rs +++ b/interactive_engine/executor/store/groot/src/db/graph/meta.rs @@ -5,7 +5,7 @@ use protobuf::Message; use super::table_manager::*; use super::types::*; -use crate::db::api::GraphErrorCode::InvalidData; +use crate::db::api::ErrorCode::INVALID_DATA; use crate::db::api::*; use crate::db::common::bytes::transform; use crate::db::common::bytes::util::parse_pb; @@ -128,7 +128,7 @@ impl Meta { "current label idx {}, create label id {}", current_label_idx, label_id ); - return Err(GraphError::new(GraphErrorCode::InvalidOperation, msg)); + return Err(GraphError::new(ErrorCode::INVALID_OPERATION, msg)); } graph_def.add_type(label_id, x.type_def.clone())?; graph_def.put_vertex_table_id(label_id, x.table_id); @@ -155,7 +155,7 @@ impl Meta { "current label idx {}, create label id {}", current_label_idx, label_id ); - return Err(GraphError::new(GraphErrorCode::InvalidOperation, msg)); + return Err(GraphError::new(ErrorCode::INVALID_OPERATION, msg)); } graph_def.add_type(label_id, x.type_def.clone())?; graph_def.set_label_idx(label_id); @@ -223,7 +223,7 @@ impl Meta { label_id, x.table_id, x.si, x.type_def ); let mut graph_def = self.graph_def_lock.lock()?; - graph_def.update_type(label_id, x.type_def.clone()); + let _ = graph_def.update_type(label_id, x.type_def.clone()); graph_def.increase_version(); vertex_manager_builder.update_type(x.si, x.label_id, &x.type_def)?; } @@ -231,7 +231,7 @@ impl Meta { let label_id = x.type_def.get_label_id(); info!("AddEdgeProperty label {:?}, si {:?}, typedef {:?}", label_id, x.si, x.type_def); let mut graph_def = self.graph_def_lock.lock()?; - graph_def.update_type(label_id, x.type_def.clone()); + let _ = graph_def.update_type(label_id, x.type_def.clone()); graph_def.increase_version(); edge_manager_builder.update_edge_type(x.si, x.label_id, &x.type_def)?; } @@ -245,7 +245,7 @@ impl Meta { let current_version = graph_def.get_version(); if current_version >= schema_version { let msg = format!("current schema version {} >= {}", current_version, schema_version); - let err = gen_graph_err!(GraphErrorCode::InvalidOperation, msg, create_vertex_type); + let err = gen_graph_err!(ErrorCode::INVALID_OPERATION, msg, create_vertex_type); return Err(err); } Ok(()) @@ -302,7 +302,7 @@ impl Meta { let current_label_idx = graph_def.get_label_idx(); if current_label_idx >= label_id { let msg = format!("current label idx {}, create label id {}", current_label_idx, label_id); - return Err(GraphError::new(GraphErrorCode::InvalidOperation, msg)); + return Err(GraphError::new(ErrorCode::INVALID_OPERATION, msg)); } graph_def.add_type(label_id, type_def.clone())?; graph_def.put_vertex_table_id(label_id, table_id); @@ -330,7 +330,7 @@ impl Meta { "Property with name '{}' already exists in type for label_id {}", new_property_name, label_id ); - return Err(GraphError::new(GraphErrorCode::InvalidOperation, msg)); + return Err(GraphError::new(ErrorCode::INVALID_OPERATION, msg)); } cloned.add_property(new_property.clone()); } @@ -338,14 +338,13 @@ impl Meta { let item = AddVertexPropertyItem::new(si, schema_version, label_id, table_id, cloned.clone()); self.write_item(item)?; { - let current_label_idx = graph_def.get_label_idx(); graph_def.update_type(label_id, cloned.clone())?; graph_def.increase_version(); } Ok((Table::new(si, table_id), cloned)) } else { let msg = format!("current label id {} not exist.", label_id); - return Err(GraphError::new(GraphErrorCode::InvalidOperation, msg)); + return Err(GraphError::new(ErrorCode::INVALID_OPERATION, msg)); } } @@ -374,7 +373,7 @@ impl Meta { let current_label_idx = graph_def.get_label_idx(); if current_label_idx >= label_id { let msg = format!("current label idx {}, create label id {}", current_label_idx, label_id); - return Err(GraphError::new(GraphErrorCode::InvalidOperation, msg)); + return Err(GraphError::new(ErrorCode::INVALID_OPERATION, msg)); } graph_def.add_type(type_def.get_label_id(), type_def.clone())?; graph_def.set_label_idx(label_id); @@ -399,7 +398,7 @@ impl Meta { "Property with name '{}' already exists in type for label_id {}", new_property_name, label_id ); - return Err(GraphError::new(GraphErrorCode::InvalidOperation, msg)); + return Err(GraphError::new(ErrorCode::INVALID_OPERATION, msg)); } cloned.add_property(new_property.clone()); } @@ -407,14 +406,13 @@ impl Meta { let item = AddEdgePropertyItem::new(si, schema_version, label_id, cloned.clone()); self.write_item(item)?; { - let current_label_idx = graph_def.get_label_idx(); graph_def.update_type(label_id, cloned.clone())?; graph_def.increase_version(); } Ok(cloned) } else { let msg = format!("current label id {} not exist.", label_id); - return Err(GraphError::new(GraphErrorCode::InvalidOperation, msg)); + return Err(GraphError::new(ErrorCode::INVALID_OPERATION, msg)); } } @@ -470,7 +468,7 @@ impl Meta { let table_id = id.to_be() + 1; if table_id >= 0 { let msg = format!("cannot assign any more table id"); - let err = gen_graph_err!(GraphErrorCode::InvalidOperation, msg, get_next_table_id); + let err = gen_graph_err!(ErrorCode::INVALID_OPERATION, msg, get_next_table_id); return Err(err); } Ok(table_id) @@ -542,7 +540,7 @@ impl MetaItem { fn common_parse_key<'a>(k: &'a [u8], prefix: &str, size: usize) -> GraphResult> { if transform::bytes_to_i64(&k[0..8])?.to_be() != META_TABLE_ID { let msg = format!("invalid meta key"); - let err = gen_graph_err!(GraphErrorCode::InvalidData, msg, common_parse_key, k, prefix, size); + let err = gen_graph_err!(ErrorCode::INVALID_DATA, msg, common_parse_key, k, prefix, size); return Err(err); } let key = res_unwrap!(transform::bytes_to_str(&k[8..]), common_parse_key, k, prefix, size)?; @@ -551,7 +549,7 @@ fn common_parse_key<'a>(k: &'a [u8], prefix: &str, size: usize) -> GraphResult { - if let TypeNotFound = e.get_error_code() { + if let TYPE_NOT_FOUND = e.get_error_code() { Box::new(::std::iter::empty()) } else { return Err(e); @@ -257,7 +257,7 @@ impl MultiVersionGraph for GraphStore { } if self.vertex_manager.contains_type(si, label_id) { let msg = format!("vertex#{} already exists", label_id); - let err = gen_graph_err!(GraphErrorCode::InvalidOperation, msg, create_vertex_type); + let err = gen_graph_err!(ErrorCode::INVALID_OPERATION, msg, create_vertex_type); return Err(err); } self.meta @@ -282,7 +282,7 @@ impl MultiVersionGraph for GraphStore { } if !self.vertex_manager.contains_type(si, label_id) { let msg = format!("vertex#{} does not exist", label_id); - let err = gen_graph_err!(GraphErrorCode::InvalidOperation, msg, add_vertex_type_properties); + let err = gen_graph_err!(ErrorCode::INVALID_OPERATION, msg, add_vertex_type_properties); return Err(err); } self.meta @@ -307,7 +307,7 @@ impl MultiVersionGraph for GraphStore { } if self.edge_manager.contains_edge(label_id) { let msg = format!("edge#{} already exists", label_id); - let err = gen_graph_err!(GraphErrorCode::InvalidOperation, msg, create_edge); + let err = gen_graph_err!(ErrorCode::INVALID_OPERATION, msg, create_edge); return Err(err); } self.meta @@ -331,7 +331,7 @@ impl MultiVersionGraph for GraphStore { } if !self.edge_manager.contains_edge(label_id) { let msg = format!("edge#{} does not exist", label_id); - let err = gen_graph_err!(GraphErrorCode::InvalidOperation, msg, add_edge_type_properties); + let err = gen_graph_err!(ErrorCode::INVALID_OPERATION, msg, add_edge_type_properties); return Err(err); } self.meta @@ -358,7 +358,7 @@ impl MultiVersionGraph for GraphStore { .contains_edge_kind(si, edge_kind) { let msg = format!("{:?} already exists", edge_kind); - let err = gen_graph_err!(GraphErrorCode::InvalidOperation, msg, add_edge_kind, si, edge_kind); + let err = gen_graph_err!(ErrorCode::INVALID_OPERATION, msg, add_edge_kind, si, edge_kind); return Err(err); } self.meta @@ -633,7 +633,7 @@ impl MultiVersionGraph for GraphStore { let graph_def = self.meta.get_graph_def().lock()?; let pb = graph_def.to_proto()?; pb.write_to_bytes() - .map_err(|e| GraphError::new(InvalidData, format!("{:?}", e))) + .map_err(|e| GraphError::new(INVALID_DATA, format!("{:?}", e))) } fn prepare_data_load( @@ -722,7 +722,7 @@ impl GraphStore { } unknown => { let msg = format!("unknown storage {}", unknown); - let err = gen_graph_err!(GraphErrorCode::NotSupported, msg, open, config, path); + let err = gen_graph_err!(ErrorCode::UNSUPPORTED_OPERATION, msg, open, config, path); Err(err) } } @@ -819,7 +819,7 @@ impl GraphStore { }); } let msg = format!("table not found at {} of vertex#{}", si, info.get_label()); - let err = gen_graph_err!(GraphErrorCode::DataNotExists, msg, do_insert_vertex_data); + let err = gen_graph_err!(ErrorCode::DATA_NOT_EXISTS, msg, do_insert_vertex_data); Err(err) } @@ -840,7 +840,7 @@ impl GraphStore { }); } let msg = format!("table not found at {} of {:?}", si, info.get_type()); - let err = gen_graph_err!(GraphErrorCode::DataNotExists, msg, do_insert_edge_data); + let err = gen_graph_err!(ErrorCode::DATA_NOT_EXISTS, msg, do_insert_edge_data); Err(err) } @@ -848,7 +848,7 @@ impl GraphStore { let guard = self.si_guard.load(Ordering::Relaxed) as SnapshotId; if si < guard { let msg = format!("si#{} is less than current si_guard#{}", si, guard); - let err = gen_graph_err!(GraphErrorCode::InvalidOperation, msg); + let err = gen_graph_err!(ErrorCode::INVALID_OPERATION, msg); return Err(err); } Ok(()) @@ -955,7 +955,7 @@ impl GraphStore { scan.into_iter() } Err(e) => { - if let TypeNotFound = e.get_error_code() { + if let TYPE_NOT_FOUND = e.get_error_code() { Box::new(::std::iter::empty()) } else { return Err(e); @@ -1071,7 +1071,7 @@ impl GraphStore { label_id, src_id, dst_id, direction ); error!("{}", msg); - Err(gen_graph_err!(GraphErrorCode::DataNotExists, msg, get_edge_by_vertex)) + Err(gen_graph_err!(ErrorCode::DATA_NOT_EXISTS, msg, get_edge_by_vertex)) } Err(err) => Err(err), } @@ -1081,7 +1081,7 @@ impl GraphStore { let statistics = self.get_statistics(si)?; let pb = statistics.to_proto()?; pb.write_to_bytes() - .map_err(|e| GraphError::new(InvalidData, format!("{:?}", e))) + .map_err(|e| GraphError::new(INVALID_DATA, format!("{:?}", e))) } pub fn get_statistics(&self, si: SnapshotId) -> GraphResult { diff --git a/interactive_engine/executor/store/groot/src/db/graph/table_manager.rs b/interactive_engine/executor/store/groot/src/db/graph/table_manager.rs index af9165087066..df370d9db85d 100644 --- a/interactive_engine/executor/store/groot/src/db/graph/table_manager.rs +++ b/interactive_engine/executor/store/groot/src/db/graph/table_manager.rs @@ -67,7 +67,7 @@ impl Table { pub fn from_bytes(bytes: &[u8]) -> GraphResult { if bytes.len() < 4 { let msg = format!("invalid bytes"); - let err = gen_graph_err!(GraphErrorCode::InvalidData, msg, from_bytes); + let err = gen_graph_err!(ErrorCode::INVALID_DATA, msg, from_bytes); return Err(err); } let reader = UnsafeBytesReader::new(bytes); @@ -81,12 +81,12 @@ impl Table { return Ok(ret); } let msg = format!("invalid bytes"); - let err = gen_graph_err!(GraphErrorCode::InvalidData, msg, from_bytes); + let err = gen_graph_err!(ErrorCode::INVALID_DATA, msg, from_bytes); Err(err) } x => { let msg = format!("unknown encoding version {}", x); - let err = gen_graph_err!(GraphErrorCode::InvalidData, msg, from_bytes); + let err = gen_graph_err!(ErrorCode::INVALID_DATA, msg, from_bytes); return Err(err); } } diff --git a/interactive_engine/executor/store/groot/src/db/graph/tests/helper.rs b/interactive_engine/executor/store/groot/src/db/graph/tests/helper.rs index d7c556bd6ccd..89e5c9bac48c 100644 --- a/interactive_engine/executor/store/groot/src/db/graph/tests/helper.rs +++ b/interactive_engine/executor/store/groot/src/db/graph/tests/helper.rs @@ -407,7 +407,7 @@ impl<'a, G: MultiVersionGraph> GraphTestHelper<'a, G> { fn check_and_update_si(&mut self, si: SnapshotId) -> GraphResult<()> { if si < self.cur_si { let msg = format!("si#{} is less than cur_si#{}", si, self.cur_si); - let err = gen_graph_err!(GraphErrorCode::InvalidOperation, msg, check_and_update_si, si); + let err = gen_graph_err!(ErrorCode::INVALID_OPERATION, msg, check_and_update_si, si); return Err(err); } self.cur_si = si; diff --git a/interactive_engine/executor/store/groot/src/db/graph/types/edge.rs b/interactive_engine/executor/store/groot/src/db/graph/types/edge.rs index 78f1c6a2e1ab..763ffd0f7e20 100644 --- a/interactive_engine/executor/store/groot/src/db/graph/types/edge.rs +++ b/interactive_engine/executor/store/groot/src/db/graph/types/edge.rs @@ -385,11 +385,11 @@ impl EdgeManagerInner { } } let msg = format!("no {:?} is alive at {}", kind, si); - let err = gen_graph_err!(GraphErrorCode::TypeNotFound, msg, get_edge_kind, si, kind); + let err = gen_graph_err!(ErrorCode::TYPE_NOT_FOUND, msg, get_edge_kind, si, kind); return Err(err); } let msg = format!("edge {:?} not found", kind); - let err = gen_graph_err!(GraphErrorCode::TypeNotFound, msg, get_edge_kind, si, kind); + let err = gen_graph_err!(ErrorCode::TYPE_NOT_FOUND, msg, get_edge_kind, si, kind); Err(err) } @@ -400,11 +400,11 @@ impl EdgeManagerInner { return Ok(info.clone()); } let msg = format!("edge#{} is not alive at {}", label, si); - let err = gen_graph_err!(GraphErrorCode::TypeNotFound, msg, get_edge_info, si, label); + let err = gen_graph_err!(ErrorCode::TYPE_NOT_FOUND, msg, get_edge_info, si, label); return Err(err); } let msg = format!("edge#{} not found", label); - let err = gen_graph_err!(GraphErrorCode::TypeNotFound, msg, get_edge_info, si, label); + let err = gen_graph_err!(ErrorCode::TYPE_NOT_FOUND, msg, get_edge_info, si, label); Err(err) } @@ -426,7 +426,7 @@ impl EdgeManagerInner { fn create_edge_type(&mut self, si: SnapshotId, label: LabelId, type_def: &TypeDef) -> GraphResult<()> { if self.info_map.contains_key(&label) { let msg = format!("edge#{} already exists", label); - let err = gen_graph_err!(GraphErrorCode::InvalidOperation, msg, create_edge_type); + let err = gen_graph_err!(ErrorCode::INVALID_OPERATION, msg, create_edge_type); return Err(err); } let info = EdgeInfo::new(si, label); @@ -440,7 +440,7 @@ impl EdgeManagerInner { fn update_edge_type(&mut self, si: SnapshotId, label: LabelId, type_def: &TypeDef) -> GraphResult<()> { if !self.info_map.contains_key(&label) { let msg = format!("edge#{} not found.", label); - let err = gen_graph_err!(GraphErrorCode::InvalidOperation, msg, update_edge_type); + let err = gen_graph_err!(ErrorCode::INVALID_OPERATION, msg, update_edge_type); return Err(err); } if let Some(info) = self.info_map.get(&label) { @@ -460,7 +460,7 @@ impl EdgeManagerInner { return Ok(()); } let msg = format!("edge#{} not found", label); - let err = gen_graph_err!(GraphErrorCode::TypeNotFound, msg, drop_edge_type, si, label); + let err = gen_graph_err!(ErrorCode::TYPE_NOT_FOUND, msg, drop_edge_type, si, label); Err(err) } @@ -478,7 +478,7 @@ impl EdgeManagerInner { return Ok(()); } let msg = format!("edge#{} not found", kind.edge_label_id); - let err = gen_graph_err!(GraphErrorCode::TypeNotFound, msg, get_edge_kind, si, kind); + let err = gen_graph_err!(ErrorCode::TYPE_NOT_FOUND, msg, get_edge_kind, si, kind); Err(err) } @@ -488,7 +488,7 @@ impl EdgeManagerInner { return Ok(()); } let msg = format!("edge#{:?} not found", kind); - let err = gen_graph_err!(GraphErrorCode::TypeNotFound, msg, remove_edge_kind, si, kind); + let err = gen_graph_err!(ErrorCode::TYPE_NOT_FOUND, msg, remove_edge_kind, si, kind); Err(err) } diff --git a/interactive_engine/executor/store/groot/src/db/graph/types/vertex.rs b/interactive_engine/executor/store/groot/src/db/graph/types/vertex.rs index 54dbf69a2ea0..3238079be7db 100644 --- a/interactive_engine/executor/store/groot/src/db/graph/types/vertex.rs +++ b/interactive_engine/executor/store/groot/src/db/graph/types/vertex.rs @@ -108,7 +108,7 @@ impl VertexTypeManager { ) -> GraphResult<()> { if si != table0.start_si { let msg = format!("type si {} must be equal to table0.start_si {}", si, table0.start_si); - let err = gen_graph_err!(GraphErrorCode::InvalidOperation, msg, create_type); + let err = gen_graph_err!(ErrorCode::INVALID_OPERATION, msg, create_type); return Err(err); } let guard = &epoch::pin(); @@ -116,12 +116,12 @@ impl VertexTypeManager { let mut map_clone = unsafe { map.as_ref() } .ok_or_else(|| { let msg = "get map reference return `None`".to_string(); - gen_graph_err!(GraphErrorCode::InvalidData, msg, get_map, si, label) + gen_graph_err!(ErrorCode::INVALID_DATA, msg, get_map, si, label) })? .clone(); if map_clone.contains_key(&label) { let msg = format!("vertex#{} already exists", label); - let err = gen_graph_err!(GraphErrorCode::InvalidOperation, msg, create_type); + let err = gen_graph_err!(ErrorCode::INVALID_OPERATION, msg, create_type); return Err(err); } let info = VertexTypeInfo::new(si, label); @@ -139,20 +139,20 @@ impl VertexTypeManager { ) -> GraphResult<()> { if si != table0.start_si { let msg = format!("type si {} must be equal to table0.start_si {}", si, table0.start_si); - let err = gen_graph_err!(GraphErrorCode::InvalidOperation, msg, update_type); + let err = gen_graph_err!(ErrorCode::INVALID_OPERATION, msg, update_type); return Err(err); } let guard = &epoch::pin(); let map = self.get_map(guard); - let mut map_clone = unsafe { map.as_ref() } + let map_clone = unsafe { map.as_ref() } .ok_or_else(|| { let msg = "get map reference return `None`".to_string(); - gen_graph_err!(GraphErrorCode::InvalidData, msg, get_map, si, label) + gen_graph_err!(ErrorCode::INVALID_DATA, msg, get_map, si, label) })? .clone(); if !map_clone.contains_key(&label) { let msg = format!("vertex#{} not found.", label); - let err = gen_graph_err!(GraphErrorCode::InvalidOperation, msg, update_type); + let err = gen_graph_err!(ErrorCode::INVALID_OPERATION, msg, update_type); return Err(err); } if let Some(info) = map_clone.get(&label) { @@ -169,18 +169,18 @@ impl VertexTypeManager { let map = self.get_map(&guard); let map_ref = unsafe { map.as_ref() }.ok_or_else(|| { let msg = "get map reference return `None`".to_string(); - gen_graph_err!(GraphErrorCode::InvalidData, msg, get_map, si, label) + gen_graph_err!(ErrorCode::INVALID_DATA, msg, get_map, si, label) })?; if let Some(info) = map_ref.get(&label) { if info.is_alive_at(si) { return Ok(info.clone()); } let msg = format!("vertex#{} is not visible at {}", label, si); - let err = gen_graph_err!(GraphErrorCode::TypeNotFound, msg, get, si, label); + let err = gen_graph_err!(ErrorCode::TYPE_NOT_FOUND, msg, get, si, label); return Err(err); } let msg = format!("vertex#{} not found", label); - let err = gen_graph_err!(GraphErrorCode::TypeNotFound, msg, get, si, label); + let err = gen_graph_err!(ErrorCode::TYPE_NOT_FOUND, msg, get, si, label); Err(err) } @@ -189,7 +189,7 @@ impl VertexTypeManager { let map = self.get_map(&guard); let map_ref = unsafe { map.as_ref() }.ok_or_else(|| { let msg = "get map reference return `None`".to_string(); - gen_graph_err!(GraphErrorCode::InvalidData, msg, get_map, si, label) + gen_graph_err!(ErrorCode::INVALID_DATA, msg, get_map, si, label) })?; if let Some(info) = map_ref.get(&label) { if info.is_alive_at(si) { @@ -197,11 +197,11 @@ impl VertexTypeManager { return Ok(ret); } let msg = format!("vertex#{} is not visible at {}", label, si); - let err = gen_graph_err!(GraphErrorCode::TypeNotFound, msg, get, si, label); + let err = gen_graph_err!(ErrorCode::TYPE_NOT_FOUND, msg, get, si, label); return Err(err); } let msg = format!("vertex#{} not found", label); - let err = gen_graph_err!(GraphErrorCode::TypeNotFound, msg, get, si, label); + let err = gen_graph_err!(ErrorCode::TYPE_NOT_FOUND, msg, get, si, label); Err(err) } @@ -210,7 +210,7 @@ impl VertexTypeManager { let map = self.get_map(&guard); let map_ref = unsafe { map.as_ref() }.ok_or_else(|| { let msg = "get map reference return `None`".to_string(); - gen_graph_err!(GraphErrorCode::InvalidData, msg, get_map, si, label) + gen_graph_err!(ErrorCode::INVALID_DATA, msg, get_map, si, label) })?; if let Some(info) = map_ref.get(&label) { info.lifetime.set_end(si); @@ -223,7 +223,7 @@ impl VertexTypeManager { let map = self.get_map(guard); let map_ref: &VertexMap = unsafe { map.as_ref() }.ok_or_else(|| { let msg = "get map reference return `None`".to_string(); - gen_graph_err!(GraphErrorCode::InvalidData, msg, get_map, si) + gen_graph_err!(ErrorCode::INVALID_DATA, msg, get_map, si) })?; let mut b = Vec::new(); let mut table_ids = Vec::new(); @@ -262,7 +262,7 @@ impl VertexTypeManagerBuilder { pub fn create(&mut self, si: SnapshotId, label: LabelId, type_def: &TypeDef) -> GraphResult<()> { if self.map.contains_key(&label) { let msg = format!("vertex#{} already exists", label); - let err = gen_graph_err!(GraphErrorCode::InvalidOperation, msg, create, si, label, type_def); + let err = gen_graph_err!(ErrorCode::INVALID_OPERATION, msg, create, si, label, type_def); return Err(err); } let info = VertexTypeInfo::new(si, label); @@ -275,7 +275,7 @@ impl VertexTypeManagerBuilder { pub fn update_type(&mut self, si: SnapshotId, label: LabelId, type_def: &TypeDef) -> GraphResult<()> { if !self.map.contains_key(&label) { let msg = format!("vertex#{} not found.", label); - let err = gen_graph_err!(GraphErrorCode::InvalidOperation, msg, create_type); + let err = gen_graph_err!(ErrorCode::INVALID_OPERATION, msg, create_type); return Err(err); } if let Some(info) = self.map.get(&label) { @@ -291,7 +291,7 @@ impl VertexTypeManagerBuilder { return Ok(()); } let msg = format!("vertex#{} not found", label); - let err = gen_graph_err!(GraphErrorCode::TypeNotFound, msg, drop, si, label); + let err = gen_graph_err!(ErrorCode::TYPE_NOT_FOUND, msg, drop, si, label); Err(err) } @@ -302,7 +302,7 @@ impl VertexTypeManagerBuilder { } } let msg = format!("vertex#{} not found", label); - let err = gen_graph_err!(GraphErrorCode::TypeNotFound, msg, get_info_mut, label); + let err = gen_graph_err!(ErrorCode::TYPE_NOT_FOUND, msg, get_info_mut, label); Err(err) } diff --git a/interactive_engine/executor/store/groot/src/db/graph/version.rs b/interactive_engine/executor/store/groot/src/db/graph/version.rs index 7a3613104afe..1f7f367f7588 100644 --- a/interactive_engine/executor/store/groot/src/db/graph/version.rs +++ b/interactive_engine/executor/store/groot/src/db/graph/version.rs @@ -68,7 +68,7 @@ impl VersionManager { pub fn add(&self, si: SnapshotId, data: i64) -> GraphResult<()> { if data == TOMBSTONE { let msg = format!("cannot using this interface to add tombstone"); - let err = gen_graph_err!(GraphErrorCode::InvalidOperation, msg, add, si, data); + let err = gen_graph_err!(ErrorCode::INVALID_OPERATION, msg, add, si, data); return Err(err); } let res = self.do_add_data(si, data); @@ -101,7 +101,7 @@ impl VersionManager { let _data = self.slots[idx].get_data(); if cur_si == 0 { let msg = format!("cur_si should not be 0! Maybe thread unsafe"); - let err = gen_graph_err!(GraphErrorCode::GraphStoreBug, msg, gc, si); + let err = gen_graph_err!(ErrorCode::ILLEGAL_STATE, msg, gc, si); return Err(err); } if cur_si <= si { @@ -127,13 +127,13 @@ impl VersionManager { fn do_add_data(&self, si: SnapshotId, data: i64) -> GraphResult<()> { if si <= 0 { let msg = format!("cannot add data with version less equal than 0"); - let err = gen_graph_err!(GraphErrorCode::InvalidOperation, msg, do_add_data, si, data); + let err = gen_graph_err!(ErrorCode::INVALID_OPERATION, msg, do_add_data, si, data); return Err(err); } let _guard = res_unwrap!(self.lock.lock(), add, si, data)?; if self.size() >= MAX_SIZE { let msg = format!("version count exceed limit {}", MAX_SIZE); - let err = gen_graph_err!(GraphErrorCode::TooManyVersions, msg, do_add_data, si, data); + let err = gen_graph_err!(ErrorCode::TOO_MANY_VERSIONS, msg, do_add_data, si, data); return Err(err); } let tail = self.get_tail(); @@ -145,7 +145,7 @@ impl VersionManager { if last_si >= si { let msg = format!("version {} is less equal than last version {}, it's invalid", si, last_si); - let err = gen_graph_err!(GraphErrorCode::InvalidOperation, msg, do_add_data, si, data); + let err = gen_graph_err!(ErrorCode::INVALID_OPERATION, msg, do_add_data, si, data); return Err(err); } } diff --git a/interactive_engine/executor/store/groot/src/db/proto/mod.rs b/interactive_engine/executor/store/groot/src/db/proto/mod.rs index 0c6572aa77d1..6b05a0fcc8d5 100644 --- a/interactive_engine/executor/store/groot/src/db/proto/mod.rs +++ b/interactive_engine/executor/store/groot/src/db/proto/mod.rs @@ -4,3 +4,5 @@ pub mod schema_common; pub mod model; #[rustfmt::skip] pub mod schema; +#[rustfmt::skip] +pub mod insight; diff --git a/interactive_engine/executor/store/groot/src/db/storage/rocksdb.rs b/interactive_engine/executor/store/groot/src/db/storage/rocksdb.rs index 26b704972de6..50488abb6983 100644 --- a/interactive_engine/executor/store/groot/src/db/storage/rocksdb.rs +++ b/interactive_engine/executor/store/groot/src/db/storage/rocksdb.rs @@ -32,7 +32,7 @@ impl RocksDB { .expect("invalid config, missing store.data.path"); let db = DB::open(&opts, path).map_err(|e| { let msg = format!("open rocksdb at {} failed: {}", path, e.into_string()); - gen_graph_err!(GraphErrorCode::ExternalStorageError, msg, open, options, path) + gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg, open, options, path) })?; let ret = RocksDB { db: Atomic::new(Arc::new(db)), options: options.clone(), is_secondary: false }; Ok(ret) @@ -41,7 +41,7 @@ impl RocksDB { pub fn open_as_secondary(options: &HashMap) -> GraphResult { let db = RocksDB::open_helper(options, false).map_err(|e| { let msg = format!("open rocksdb at {:?}, error: {:?}", options, e); - gen_graph_err!(GraphErrorCode::ExternalStorageError, msg, open_as_secondary) + gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg, open_as_secondary) })?; let ret = RocksDB { db: Atomic::new(Arc::new(db)), options: options.clone(), is_secondary: true }; @@ -107,13 +107,13 @@ impl RocksDB { Ok(None) => Ok(None), Err(e) => { let msg = format!("rocksdb.get failed because {}", e.into_string()); - let err = gen_graph_err!(GraphErrorCode::ExternalStorageError, msg); + let err = gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg); Err(err) } } } else { let msg = format!("rocksdb.get failed because the acquired db is `None`"); - let err = gen_graph_err!(GraphErrorCode::ExternalStorageError, msg); + let err = gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg); Err(err) } } @@ -128,11 +128,11 @@ impl RocksDB { if let Some(db) = unsafe { db_shared.as_ref() } { db.put(key, val).map_err(|e| { let msg = format!("rocksdb.put failed because {}", e.into_string()); - gen_graph_err!(GraphErrorCode::ExternalStorageError, msg) + gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg) }) } else { let msg = format!("rocksdb.put failed because the acquired db is `None`"); - let err = gen_graph_err!(GraphErrorCode::ExternalStorageError, msg); + let err = gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg); Err(err) } } @@ -147,11 +147,11 @@ impl RocksDB { if let Some(db) = unsafe { db_shared.as_ref() } { db.delete(key).map_err(|e| { let msg = format!("rocksdb.delete failed because {}", e.into_string()); - gen_graph_err!(GraphErrorCode::ExternalStorageError, msg) + gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg) }) } else { let msg = format!("rocksdb.delete failed because the acquired db is `None`"); - let err = gen_graph_err!(GraphErrorCode::ExternalStorageError, msg); + let err = gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg); Err(err) } } @@ -163,7 +163,7 @@ impl RocksDB { Ok(StorageIter::RocksDB(RocksDBIter::new_prefix(db.clone(), prefix, guard))) } else { let msg = format!("rocksdb.scan_prefix failed because the acquired db is `None`"); - let err = gen_graph_err!(GraphErrorCode::ExternalStorageError, msg); + let err = gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg); Err(err) } } @@ -175,7 +175,7 @@ impl RocksDB { Ok(StorageIter::RocksDB(RocksDBIter::new_start(db.clone(), start, guard))) } else { let msg = format!("rocksdb.scan_from failed because the acquired db is `None`"); - let err = gen_graph_err!(GraphErrorCode::ExternalStorageError, msg); + let err = gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg); Err(err) } } @@ -187,7 +187,7 @@ impl RocksDB { Ok(StorageIter::RocksDB(RocksDBIter::new_range(db.clone(), start, end, guard))) } else { let msg = format!("rocksdb.new_range failed because the acquired db is `None`"); - let err = gen_graph_err!(GraphErrorCode::ExternalStorageError, msg); + let err = gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg); Err(err) } } @@ -205,7 +205,7 @@ impl RocksDB { batch.delete_range(start, end); db.write(batch).map_err(|e| { let msg = format!("rocksdb.delete_range failed because {}", e.into_string()); - gen_graph_err!(GraphErrorCode::ExternalStorageError, msg) + gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg) })?; let mut val = false; if let Some(conf_str) = self @@ -220,7 +220,7 @@ impl RocksDB { Ok(()) } else { let msg = format!("rocksdb.delete_range failed because the acquired db is `None`"); - let err = gen_graph_err!(GraphErrorCode::ExternalStorageError, msg); + let err = gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg); Err(err) } } @@ -240,7 +240,7 @@ impl RocksDB { Ok(()) } else { let msg = format!("rocksdb.compact failed because the acquired db is `None`"); - let err = gen_graph_err!(GraphErrorCode::ExternalStorageError, msg); + let err = gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg); Err(err) } } @@ -258,11 +258,11 @@ impl RocksDB { db.ingest_external_file_opts(&options, files.to_vec()) .map_err(|e| { let msg = format!("rocksdb.load file {:?} failed because {}", files, e.into_string()); - gen_graph_err!(GraphErrorCode::ExternalStorageError, msg) + gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg) }) } else { let msg = format!("rocksdb.load failed because the acquired db is `None`"); - let err = gen_graph_err!(GraphErrorCode::ExternalStorageError, msg); + let err = gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg); Err(err) } } @@ -274,11 +274,11 @@ impl RocksDB { backup_path.to_string(), e.into_string() ); - gen_graph_err!(GraphErrorCode::ExternalStorageError, msg) + gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg) })?; let env = Env::new().map_err(|e| { let msg = format!("Gen rocksdb Env failed because {}", e.into_string()); - gen_graph_err!(GraphErrorCode::ExternalStorageError, msg) + gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg) })?; let backup_engine = BackupEngine::open(&backup_opts, &env).map_err(|e| { let msg = format!( @@ -286,7 +286,7 @@ impl RocksDB { backup_path.to_string(), e.into_string() ); - gen_graph_err!(GraphErrorCode::ExternalStorageError, msg) + gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg) })?; let guard = epoch::pin(); let db_shared = self.get_db(&guard); @@ -295,7 +295,7 @@ impl RocksDB { Ok(Box::from(ret)) } else { let msg = format!("open rocksdb backup engine failed because the acquired db is `None`"); - let err = gen_graph_err!(GraphErrorCode::ExternalStorageError, msg); + let err = gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg); Err(err) } } @@ -307,7 +307,7 @@ impl RocksDB { Ok(Box::new(Scan::new(db.clone(), prefix, guard))) } else { let msg = format!("rocksdb.new_scan failed because the acquired db is `None`"); - let err = gen_graph_err!(GraphErrorCode::ExternalStorageError, msg); + let err = gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg); Err(err) } } @@ -321,11 +321,11 @@ impl RocksDB { if let Some(db) = unsafe { db_shared.as_ref() } { db.try_catch_up_with_primary().map_err(|e| { let msg = format!("rocksdb.try_catch_up_with_primary failed because {:?}", e); - gen_graph_err!(GraphErrorCode::ExternalStorageError, msg) + gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg) }) } else { let msg = format!("rocksdb.try_catch_up_with_primary failed because the acquired db is `None`"); - let err = gen_graph_err!(GraphErrorCode::ExternalStorageError, msg); + let err = gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg); Err(err) } } @@ -338,7 +338,7 @@ impl RocksDB { std::thread::sleep(Duration::from_secs(wait_sec)); let db = RocksDB::open_helper(&self.options, true).map_err(|e| { let msg = format!("open rocksdb at {:?}, error: {:?}", self.options, e); - gen_graph_err!(GraphErrorCode::ExternalStorageError, msg, open_as_secondary) + gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg, open_as_secondary) })?; let ret = db.try_catch_up_with_primary(); if ret.is_err() { @@ -381,12 +381,12 @@ impl RocksDBBackupEngine { .create_new_backup(&self.db) .map_err(|e| { let msg = format!("create new rocksdb backup failed, because {}", e.into_string()); - gen_graph_err!(GraphErrorCode::ExternalStorageError, msg) + gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg) })?; let after = self.get_backup_list(); if after.len() != before.len() + 1 { let msg = "get new created rocksdb backup id failed".to_string(); - return Err(gen_graph_err!(GraphErrorCode::ExternalStorageError, msg)); + return Err(gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg)); } let new_backup_id = *after.iter().max().unwrap(); Ok(new_backup_id) @@ -410,7 +410,7 @@ impl RocksDBBackupEngine { backup_id, e.into_string() ); - gen_graph_err!(GraphErrorCode::ExternalStorageError, msg) + gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg) })?; Ok(()) } @@ -421,7 +421,7 @@ impl RocksDBBackupEngine { .map_err(|e| { let msg = format!("rocksdb backup {} verify failed, because {}", backup_id, e.into_string()); - gen_graph_err!(GraphErrorCode::ExternalStorageError, msg) + gen_graph_err!(ErrorCode::EXTERNAL_STORAGE_ERROR, msg) })?; Ok(()) } diff --git a/interactive_engine/executor/store/groot/src/db/util/lock.rs b/interactive_engine/executor/store/groot/src/db/util/lock.rs index 81ecc6909bda..7feaefce314a 100644 --- a/interactive_engine/executor/store/groot/src/db/util/lock.rs +++ b/interactive_engine/executor/store/groot/src/db/util/lock.rs @@ -18,7 +18,7 @@ impl GraphMutexLock { .map(|guard| GraphMutexLockGuard::new(guard)) .map_err(|e| { let msg = format!("{:?}", e); - gen_graph_err!(GraphErrorCode::LockFailed, msg, lock) + gen_graph_err!(ErrorCode::LOCK_FAILED, msg, lock) }) } } diff --git a/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardIrMetaReader.java b/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardIrMetaReader.java index fa550eec6571..90e6e500a838 100644 --- a/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardIrMetaReader.java +++ b/interactive_engine/frontend/src/main/java/com/alibaba/graphscope/frontend/VineyardIrMetaReader.java @@ -22,12 +22,12 @@ import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.meta.reader.IrMetaReader; import com.alibaba.graphscope.common.ir.meta.schema.IrGraphSchema; +import com.alibaba.graphscope.groot.common.exception.UnimplementedException; import com.alibaba.graphscope.groot.common.schema.api.GraphSchema; import com.alibaba.graphscope.groot.common.schema.api.GraphStatistics; import com.alibaba.graphscope.groot.common.schema.impl.DefaultGraphSchema; import org.apache.commons.io.FileUtils; -import org.apache.commons.lang3.NotImplementedException; import java.io.File; import java.io.IOException; @@ -51,9 +51,9 @@ public IrMeta readMeta() throws IOException { } @Override - public GraphStatistics readStats(GraphId graphId) throws IOException { + public GraphStatistics readStats(GraphId graphId) { // TODO: support statistics, otherwise, the CBO will not work - throw new NotImplementedException( + throw new UnimplementedException( "reading graph statistics in vineyard is unimplemented yet"); } } diff --git a/interactive_engine/groot-client/pom.xml b/interactive_engine/groot-client/pom.xml index 05e0c3e27b3c..6e98a5525928 100644 --- a/interactive_engine/groot-client/pom.xml +++ b/interactive_engine/groot-client/pom.xml @@ -98,6 +98,7 @@ ddl_service.proto write_service.proto request_option.proto + error/insight.proto diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/Utils.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/Utils.java index 301b016dac90..3bcdaddea5ee 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/Utils.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/Utils.java @@ -18,6 +18,7 @@ import com.alibaba.graphscope.groot.common.config.Configs; import com.alibaba.graphscope.groot.common.config.DiscoveryConfig; import com.alibaba.graphscope.groot.common.config.StoreConfig; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.groot.operation.OperationBatch; import com.alibaba.graphscope.groot.operation.OperationBlob; import com.alibaba.graphscope.groot.operation.OperationType; @@ -61,7 +62,7 @@ public static String getHostTemplate(Configs configs, RoleType role) { svcName = String.format(svcTemplate, releaseName, "store"); break; default: - throw new IllegalArgumentException("invalid role [" + role + "]"); + throw new InvalidArgumentException("invalid role [" + role + "]"); } String dnsTemplate = "%s-{}.%s-headless"; return String.format(dnsTemplate, svcName, svcName); @@ -97,18 +98,18 @@ public static int getPort(Configs configs, RoleType role, int idx) { s = CommonConfig.GAIA_ENGINE_PORT.get(configs); break; default: - throw new IllegalArgumentException("invalid role [" + role + "]"); + throw new InvalidArgumentException("invalid role [" + role + "]"); } if (s.isEmpty()) { // For backward compatibility return CommonConfig.RPC_PORT.get(configs); } else { String[] array = s.split(","); if (idx >= array.length) { - // throw new IllegalArgumentException("Invalid index " + idx + " of " + s); + // throw new InvalidArgumentException("Invalid index " + idx + " of " + s); idx = 0; // Just use the first one. In this case, assume they are in different pods. } if (array[idx].isEmpty()) { - throw new IllegalArgumentException("Invalid port " + array[idx] + " of " + role); + throw new InvalidArgumentException("Invalid port " + array[idx] + " of " + role); } return Integer.parseInt(array[idx]); } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/CoordinatorSnapshotClient.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/CoordinatorSnapshotClient.java index 6d1d291479d0..35932ec189f9 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/CoordinatorSnapshotClient.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/CoordinatorSnapshotClient.java @@ -1,5 +1,6 @@ package com.alibaba.graphscope.groot.coordinator; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.groot.rpc.RpcChannel; import com.alibaba.graphscope.groot.rpc.RpcClient; import com.alibaba.graphscope.proto.groot.*; @@ -24,7 +25,8 @@ public void synchronizeSnapshot(long snapshotId) throws RuntimeException { SynchronizeMinQuerySnapshotIdRequest.newBuilder().setSnapshotId(snapshotId).build(); SynchronizeMinQuerySnapshotIdResponse res = getStub().synchronizeMinQuerySnapshotId(req); if (!res.getSuccess()) { - throw new RuntimeException("Synchronize snapshot to store failed: " + res.getErrMsg()); + throw new InvalidArgumentException( + "Synchronize snapshot to store failed: " + res.getErrMsg()); } } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/GraphInitializer.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/GraphInitializer.java index a51a00cf264b..1a51f3e99bb3 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/GraphInitializer.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/GraphInitializer.java @@ -14,7 +14,7 @@ package com.alibaba.graphscope.groot.coordinator; import com.alibaba.graphscope.groot.common.config.*; -import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.groot.common.exception.InternalException; import com.alibaba.graphscope.groot.common.util.BackupInfo; import com.alibaba.graphscope.groot.coordinator.backup.BackupManager; import com.alibaba.graphscope.groot.meta.MetaStore; @@ -52,7 +52,7 @@ public void initializeIfNeeded() { initializeZkIfNeeded(); initializeMetaIfNeeded(); } catch (Exception e) { - throw new GrootException(e); + throw new InternalException(e); } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/IdAllocator.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/IdAllocator.java index 130583aaa3b5..47fc60d5aabd 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/IdAllocator.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/IdAllocator.java @@ -1,10 +1,10 @@ package com.alibaba.graphscope.groot.coordinator; -import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.groot.common.exception.InternalException; +import com.alibaba.graphscope.groot.common.exception.NotFoundException; import com.alibaba.graphscope.groot.meta.MetaStore; import com.fasterxml.jackson.databind.ObjectMapper; -import java.io.FileNotFoundException; import java.io.IOException; public class IdAllocator { @@ -24,7 +24,7 @@ public void start() { try { recover(); } catch (IOException e) { - throw new GrootException(e); + throw new InternalException(e); } } @@ -34,7 +34,7 @@ public void stop() { private void recover() throws IOException { if (!this.metaStore.exists(ID_ALLOCATE_INFO_PATH)) { - throw new FileNotFoundException(ID_ALLOCATE_INFO_PATH); + throw new NotFoundException("File not found: " + ID_ALLOCATE_INFO_PATH); } byte[] b = this.metaStore.read(ID_ALLOCATE_INFO_PATH); this.tailId = this.objectMapper.readValue(b, Long.class); diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/SchemaService.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/SchemaService.java index 6453b3154d8b..579003af4e01 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/SchemaService.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/SchemaService.java @@ -14,7 +14,7 @@ package com.alibaba.graphscope.groot.coordinator; import com.alibaba.graphscope.groot.CompletionCallback; -import com.alibaba.graphscope.groot.schema.request.DdlException; +import com.alibaba.graphscope.groot.common.exception.DdlException; import com.alibaba.graphscope.groot.schema.request.DdlRequestBatch; import com.alibaba.graphscope.proto.groot.SchemaGrpc; import com.alibaba.graphscope.proto.groot.SubmitBatchDdlRequest; diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/SnapshotManager.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/SnapshotManager.java index 5e733d4c6515..cd755f7299b5 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/SnapshotManager.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/SnapshotManager.java @@ -17,8 +17,8 @@ import com.alibaba.graphscope.groot.common.config.CommonConfig; import com.alibaba.graphscope.groot.common.config.Configs; import com.alibaba.graphscope.groot.common.config.CoordinatorConfig; -import com.alibaba.graphscope.groot.common.exception.GrootException; -import com.alibaba.graphscope.groot.common.exception.ServiceNotReadyException; +import com.alibaba.graphscope.groot.common.exception.*; +import com.alibaba.graphscope.groot.common.exception.IllegalStateException; import com.alibaba.graphscope.groot.common.util.ThreadFactoryUtils; import com.alibaba.graphscope.groot.meta.MetaStore; import com.fasterxml.jackson.core.type.TypeReference; @@ -160,7 +160,7 @@ public void start() { try { recover(); } catch (IOException e) { - throw new GrootException(e); + throw new InternalException(e); } this.increaseWriteSnapshotIdScheduler = @@ -219,7 +219,7 @@ public void stop() { private void checkMetaPath(String path) throws FileNotFoundException { if (!this.metaStore.exists(path)) { - throw new FileNotFoundException(path); + throw new NotFoundException("File not found: " + path); } } @@ -276,7 +276,7 @@ public synchronized void commitSnapshotId( // Queue count of each store is fixed: 1 if (1 != offsets.size()) { String msg = String.format("committed offset is %s", offsets); - throw new IllegalArgumentException(msg); + throw new InvalidArgumentException(msg); } if (v > offsets.get(0)) { return v; diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/backup/BackupManager.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/backup/BackupManager.java index f6210d967357..da6eb096d2f0 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/backup/BackupManager.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/coordinator/backup/BackupManager.java @@ -18,8 +18,7 @@ import com.alibaba.graphscope.groot.common.config.BackupConfig; import com.alibaba.graphscope.groot.common.config.CommonConfig; import com.alibaba.graphscope.groot.common.config.Configs; -import com.alibaba.graphscope.groot.common.exception.BackupException; -import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.groot.common.exception.*; import com.alibaba.graphscope.groot.common.util.BackupInfo; import com.alibaba.graphscope.groot.common.util.ThreadFactoryUtils; import com.alibaba.graphscope.groot.coordinator.QuerySnapshotListener; @@ -36,7 +35,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.FileNotFoundException; import java.io.IOException; import java.util.*; import java.util.concurrent.*; @@ -116,7 +114,7 @@ public void start() { try { recover(); } catch (IOException e) { - throw new GrootException(e); + throw new InternalException(e); } this.localListener = new LocalSnapshotListener(this.schemaManager, this.localSnapshotCache); @@ -226,7 +224,7 @@ public void deleteBackup(int globalBackupId) throws BackupException, IOException public void purgeOldBackups(int keepAliveNum) throws BackupException, IOException { checkEnable(); if (keepAliveNum <= 0) { - throw new IllegalArgumentException( + throw new InvalidArgumentException( "the input keepAliveNum should > 0, got " + keepAliveNum); } else if (keepAliveNum >= this.globalBackupIdToInfo.size()) { return; @@ -302,9 +300,9 @@ private void checkEnable() throws BackupException { } } - private void checkMetaPath(String path) throws FileNotFoundException { + private void checkMetaPath(String path) { if (!this.metaStore.exists(path)) { - throw new FileNotFoundException(path); + throw new NotFoundException("File not found: " + path); } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/discovery/DiscoveryFactory.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/discovery/DiscoveryFactory.java index 7dce53432b3b..b803f4e980a7 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/discovery/DiscoveryFactory.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/discovery/DiscoveryFactory.java @@ -16,6 +16,7 @@ import com.alibaba.graphscope.groot.CuratorUtils; import com.alibaba.graphscope.groot.common.config.CommonConfig; import com.alibaba.graphscope.groot.common.config.Configs; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import org.apache.curator.framework.CuratorFramework; @@ -45,7 +46,7 @@ public NodeDiscovery makeDiscovery(LocalNodeProvider localNodeProvider) { case "ZOOKEEPER": return new ZkDiscovery(this.configs, localNodeProvider, getCurator()); default: - throw new IllegalArgumentException( + throw new InvalidArgumentException( "invalid discovery mode [" + discoveryMode + "]"); } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/discovery/GrootNode.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/discovery/GrootNode.java index 69f477b5f265..296288be2e92 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/discovery/GrootNode.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/discovery/GrootNode.java @@ -16,7 +16,7 @@ import com.alibaba.graphscope.groot.common.RoleType; import com.alibaba.graphscope.groot.common.config.CommonConfig; import com.alibaba.graphscope.groot.common.config.Configs; -import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.groot.common.exception.NetworkFailureException; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; @@ -58,7 +58,7 @@ public static GrootNode createGraphNode(RoleType role, Configs configs, int port try { host = InetAddress.getLocalHost().getHostAddress(); } catch (UnknownHostException e) { - throw new GrootException(e); + throw new NetworkFailureException(e); } } return new GrootNode(role.getName(), idx, host, port); diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/discovery/LocalNodeProvider.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/discovery/LocalNodeProvider.java index ddcae785033e..2e3e8444d4f3 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/discovery/LocalNodeProvider.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/discovery/LocalNodeProvider.java @@ -16,7 +16,7 @@ import com.alibaba.graphscope.groot.common.RoleType; import com.alibaba.graphscope.groot.common.config.CommonConfig; import com.alibaba.graphscope.groot.common.config.Configs; -import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.groot.common.exception.UnsupportedOperationException; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; @@ -43,7 +43,7 @@ public GrootNode apply(Integer port) { null, GrootNode.createGraphNode(roleType, configs, port)); if (!suc) { if (!CommonConfig.DISCOVERY_MODE.get(this.configs).equalsIgnoreCase("file")) { - throw new GrootException("localNode can only be set once"); + throw new UnsupportedOperationException("localNode can only be set once"); } } return localNodeRef.get(); diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/discovery/ZkDiscovery.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/discovery/ZkDiscovery.java index 137244c2a617..705e62061eba 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/discovery/ZkDiscovery.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/discovery/ZkDiscovery.java @@ -16,7 +16,7 @@ import com.alibaba.graphscope.groot.common.RoleType; import com.alibaba.graphscope.groot.common.config.Configs; import com.alibaba.graphscope.groot.common.config.ZkConfig; -import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.groot.common.exception.InternalException; import com.alibaba.graphscope.groot.common.util.ThreadFactoryUtils; import org.apache.curator.framework.CuratorFramework; @@ -115,7 +115,7 @@ public void start() { this.serviceCaches.add(serviceCache); } } catch (Exception e) { - throw new GrootException(e); + throw new InternalException(e); } logger.info("ZkDiscovery started"); } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/BatchDdlClient.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/BatchDdlClient.java index d755fccdaf24..3deeff2ab380 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/BatchDdlClient.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/BatchDdlClient.java @@ -1,10 +1,10 @@ package com.alibaba.graphscope.groot.frontend; +import com.alibaba.graphscope.groot.common.exception.DdlException; import com.alibaba.graphscope.groot.common.schema.wrapper.GraphDef; import com.alibaba.graphscope.groot.common.util.UuidUtils; import com.alibaba.graphscope.groot.rpc.RoleClients; import com.alibaba.graphscope.groot.schema.ddl.DdlExecutors; -import com.alibaba.graphscope.groot.schema.request.DdlException; import com.alibaba.graphscope.groot.schema.request.DdlRequestBatch; import com.google.protobuf.InvalidProtocolBufferException; diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/ClientService.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/ClientService.java index da8ed2678ccb..48f64d721751 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/ClientService.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/ClientService.java @@ -320,6 +320,7 @@ public void ingestData( AtomicBoolean finished = new AtomicBoolean(false); for (int i = 0; i < storeCount; i++) { logger.info("Store [" + i + "] started to ingest..."); + int finalI = i; this.frontendStoreClients .getClient(i) .storeIngest( @@ -335,7 +336,8 @@ public void onCompleted(Void res) { @Override public void onError(Throwable t) { - logger.error("failed ingest", t); + logger.error( + "store {} ingest from {} failed", finalI, dataPath, t); finish(t); } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/GrootDdlService.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/GrootDdlService.java index ec95ded3ee11..0a8708f1e4e5 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/GrootDdlService.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/GrootDdlService.java @@ -13,6 +13,8 @@ */ package com.alibaba.graphscope.groot.frontend; +import com.alibaba.graphscope.groot.common.exception.IllegalStateException; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.groot.common.schema.wrapper.DataType; import com.alibaba.graphscope.groot.common.schema.wrapper.EdgeKind; import com.alibaba.graphscope.groot.common.schema.wrapper.GraphDef; @@ -222,7 +224,7 @@ private TypeDefPb toTypeDefPb(TypeDef typeDef) { builder.setTypeEnum(TypeEnumPb.EDGE); break; default: - throw new IllegalArgumentException( + throw new InvalidArgumentException( "Invalid type enum [" + typeDef.getTypeEnum() + "]"); } for (PropertyDef property : typeDef.getProperties()) { diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/SchemaClient.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/SchemaClient.java index 692f11cc981d..177e742d3d48 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/SchemaClient.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/SchemaClient.java @@ -13,9 +13,9 @@ */ package com.alibaba.graphscope.groot.frontend; +import com.alibaba.graphscope.groot.common.exception.DdlException; import com.alibaba.graphscope.groot.rpc.RpcChannel; import com.alibaba.graphscope.groot.rpc.RpcClient; -import com.alibaba.graphscope.groot.schema.request.DdlException; import com.alibaba.graphscope.proto.groot.DdlRequestBatchPb; import com.alibaba.graphscope.proto.groot.SchemaGrpc; import com.alibaba.graphscope.proto.groot.SubmitBatchDdlRequest; diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/SnapshotCache.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/SnapshotCache.java index 39b53be82dd1..4f4a0254ea83 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/SnapshotCache.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/SnapshotCache.java @@ -15,6 +15,7 @@ import com.alibaba.graphscope.groot.SnapshotListener; import com.alibaba.graphscope.groot.SnapshotWithSchema; +import com.alibaba.graphscope.groot.common.exception.IllegalStateException; import com.alibaba.graphscope.groot.common.schema.api.GraphStatistics; import com.alibaba.graphscope.groot.common.schema.wrapper.GraphDef; diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/SnapshotUpdateClient.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/SnapshotUpdateClient.java index 94fa48992339..b968b23be66b 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/SnapshotUpdateClient.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/SnapshotUpdateClient.java @@ -16,6 +16,7 @@ package com.alibaba.graphscope.groot.frontend; +import com.alibaba.graphscope.groot.common.exception.InternalException; import com.alibaba.graphscope.groot.rpc.RpcChannel; import com.alibaba.graphscope.groot.rpc.RpcClient; import com.alibaba.graphscope.proto.groot.CoordinatorSnapshotServiceGrpc; @@ -45,7 +46,7 @@ public void updateSnapshot(int frontendId, long snapshotId) throws RuntimeExcept .build(); UpdateMinQuerySnapshotIdResponse res = getStub().updateMinQuerySnapshotId(req); if (!res.getSuccess()) { - throw new RuntimeException("update snapshot fail {} " + res.getErrMsg()); + throw new InternalException("update snapshot fail {} " + res.getErrMsg()); } } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/WrappedSchemaFetcher.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/WrappedSchemaFetcher.java index 54ffe13606d6..8d5df033a6e1 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/WrappedSchemaFetcher.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/WrappedSchemaFetcher.java @@ -14,6 +14,7 @@ package com.alibaba.graphscope.groot.frontend; import com.alibaba.graphscope.groot.SnapshotWithSchema; +import com.alibaba.graphscope.groot.common.exception.UnimplementedException; import com.alibaba.graphscope.groot.common.schema.api.GraphSchema; import com.alibaba.graphscope.groot.common.schema.api.GraphStatistics; import com.alibaba.graphscope.groot.common.schema.api.SchemaFetcher; @@ -61,6 +62,6 @@ public int getPartitionNum() { @Override public int getVersion() { - throw new UnsupportedOperationException(); + throw new UnimplementedException(); } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/DefaultEdgeIdGenerator.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/DefaultEdgeIdGenerator.java index e427d9eaf214..39d62db8a028 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/DefaultEdgeIdGenerator.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/DefaultEdgeIdGenerator.java @@ -3,6 +3,7 @@ import com.alibaba.graphscope.groot.common.RoleType; import com.alibaba.graphscope.groot.common.config.CommonConfig; import com.alibaba.graphscope.groot.common.config.Configs; +import com.alibaba.graphscope.groot.common.exception.UnsupportedOperationException; import com.alibaba.graphscope.groot.common.util.PkHashUtils; import com.alibaba.graphscope.groot.rpc.ChannelManager; import com.alibaba.graphscope.groot.rpc.RoleClients; @@ -40,7 +41,7 @@ public long getNextId() { @Override public long getHashId(long srcId, long dstId, int labelId, List pks) { if (pks == null || pks.size() == 0) { - throw new RuntimeException("Cannot get hash id when pk is empty"); + throw new UnsupportedOperationException("Cannot get hash id when pk is empty"); } return PkHashUtils.hash(srcId, dstId, labelId, pks); } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/GraphWriter.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/GraphWriter.java index 380a02cd15c6..4c0731dadb36 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/GraphWriter.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/GraphWriter.java @@ -2,7 +2,8 @@ import com.alibaba.graphscope.groot.CompletionCallback; import com.alibaba.graphscope.groot.common.config.Configs; -import com.alibaba.graphscope.groot.common.exception.PropertyDefNotFoundException; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; +import com.alibaba.graphscope.groot.common.exception.NotFoundException; import com.alibaba.graphscope.groot.common.schema.api.GraphElement; import com.alibaba.graphscope.groot.common.schema.api.GraphProperty; import com.alibaba.graphscope.groot.common.schema.api.GraphSchema; @@ -122,7 +123,7 @@ public void writeBatch( addClearEdgePropertiesOperation(batchBuilder, schema, dataRecord); break; default: - throw new IllegalArgumentException( + throw new InvalidArgumentException( "Invalid operationType [" + operationType + "]"); } } @@ -300,7 +301,7 @@ public static Map parseRawProperties( (propertyName, valString) -> { GraphProperty propertyDef = graphElement.getProperty(propertyName); if (propertyDef == null) { - throw new PropertyDefNotFoundException( + throw new NotFoundException( "property [" + propertyName + "] not found in [" diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/KafkaAppender.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/KafkaAppender.java index 1776aa2b003d..283f0ddd7a97 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/KafkaAppender.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/KafkaAppender.java @@ -7,7 +7,8 @@ import com.alibaba.graphscope.groot.common.config.CommonConfig; import com.alibaba.graphscope.groot.common.config.Configs; import com.alibaba.graphscope.groot.common.config.FrontendConfig; -import com.alibaba.graphscope.groot.common.exception.IngestRejectException; +import com.alibaba.graphscope.groot.common.exception.IllegalStateException; +import com.alibaba.graphscope.groot.common.exception.QueueRejectException; import com.alibaba.graphscope.groot.common.util.PartitionUtils; import com.alibaba.graphscope.groot.meta.MetaService; import com.alibaba.graphscope.groot.operation.OperationBatch; @@ -118,7 +119,7 @@ public void ingestBatch( boolean suc = this.ingestBuffer.offer(new IngestTask(requestId, operationBatch, callback)); if (!suc) { logger.warn("ingest buffer is full"); - throw new IngestRejectException("add ingestTask to buffer failed"); + throw new QueueRejectException("add ingestTask to buffer failed"); } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/WriteRequest.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/WriteRequest.java index 5eee9c78b4d2..4ce0476ef9e2 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/WriteRequest.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/frontend/write/WriteRequest.java @@ -1,5 +1,6 @@ package com.alibaba.graphscope.groot.frontend.write; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.groot.common.util.EdgeRecordKey; import com.alibaba.graphscope.groot.common.util.VertexRecordKey; import com.alibaba.graphscope.groot.operation.OperationType; @@ -46,7 +47,7 @@ public static WriteRequest parseProto(WriteRequestPb proto) { return buildWriteEdgeRequest( writeTypePb, new DataRecord(edgeRecordKey, properties)); default: - throw new IllegalArgumentException( + throw new InvalidArgumentException( "Invalid record key case [" + recordKeyCase + "]"); } } @@ -62,7 +63,7 @@ private static WriteRequest buildWriteVertexRequest(WriteTypePb typePb, DataReco case CLEAR_PROPERTY: return new WriteRequest(OperationType.CLEAR_VERTEX_PROPERTIES, dataRecord); default: - throw new IllegalArgumentException("Invalid write type [" + typePb + "]"); + throw new InvalidArgumentException("Invalid write type [" + typePb + "]"); } } @@ -77,7 +78,7 @@ private static WriteRequest buildWriteEdgeRequest(WriteTypePb typePb, DataRecord case CLEAR_PROPERTY: return new WriteRequest(OperationType.CLEAR_EDGE_PROPERTIES, dataRecord); default: - throw new IllegalArgumentException("Invalid write type [" + typePb + "]"); + throw new InvalidArgumentException("Invalid write type [" + typePb + "]"); } } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/meta/FileMetaStore.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/meta/FileMetaStore.java index 9c947c19a188..09825f1fa760 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/meta/FileMetaStore.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/meta/FileMetaStore.java @@ -13,6 +13,8 @@ */ package com.alibaba.graphscope.groot.meta; +import com.alibaba.graphscope.groot.common.exception.InvalidDataException; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -77,7 +79,8 @@ private SimpleImmutableEntry readEntry(String path) throws IOExcep int ignored = is.read(res); long realCrc = getCRC32Checksum(res); if (realCrc != crc) { - throw new IOException("Checksum mismatch for " + file.getAbsolutePath()); + throw new InvalidDataException( + "Checksum mismatch for " + file.getAbsolutePath()); } } } @@ -111,7 +114,7 @@ public byte[] read(String path) throws IOException { } else if (res1 != null) { return res1; } else { - throw new IOException("File maybe corrupted: " + path); + throw new InvalidDataException("File maybe corrupted: " + path); } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/meta/ZkMetaStore.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/meta/ZkMetaStore.java index a83a989f3afa..23915bd36258 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/meta/ZkMetaStore.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/meta/ZkMetaStore.java @@ -15,7 +15,7 @@ import com.alibaba.graphscope.groot.common.config.Configs; import com.alibaba.graphscope.groot.common.config.ZkConfig; -import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.groot.common.exception.InternalException; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.utils.ZKPaths; @@ -48,7 +48,7 @@ public boolean exists(String path) { try { stat = this.curator.checkExists().forPath(fullPath); } catch (Exception e) { - throw new GrootException(e); + throw new InternalException(e); } return stat != null; } @@ -60,7 +60,7 @@ public byte[] read(String path) throws IOException { try { data = this.curator.getData().forPath(fullPath); } catch (Exception e) { - throw new IOException(e); + throw new InternalException(e); } return data; } @@ -75,7 +75,7 @@ public void write(String path, byte[] content) throws IOException { this.curator.create().creatingParentsIfNeeded().forPath(fullPath, content); } } catch (Exception e) { - throw new IOException(e); + throw new InternalException(e); } } @@ -85,7 +85,7 @@ public void delete(String path) throws IOException { try { this.curator.delete().forPath(fullPath); } catch (Exception e) { - throw new IOException(e); + throw new InternalException(e); } } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/operation/OperationBatch.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/operation/OperationBatch.java index 921161f8271c..b57533049342 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/operation/OperationBatch.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/operation/OperationBatch.java @@ -13,6 +13,7 @@ */ package com.alibaba.graphscope.groot.operation; +import com.alibaba.graphscope.groot.common.exception.IllegalStateException; import com.alibaba.graphscope.proto.groot.OperationBatchPb; import com.alibaba.graphscope.proto.groot.OperationPb; diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/operation/OperationType.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/operation/OperationType.java index ae16576a0b0b..1a9c15b203c4 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/operation/OperationType.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/operation/OperationType.java @@ -13,6 +13,7 @@ */ package com.alibaba.graphscope.groot.operation; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.proto.groot.OpTypePb; public enum OperationType { @@ -55,7 +56,7 @@ public byte getId() { public static OperationType fromId(byte id) { if (id < 0 || id >= TYPES.length) { - throw new IllegalArgumentException("Unknown OperationType: [" + id + "]"); + throw new InvalidArgumentException("Unknown OperationType: [" + id + "]"); } return TYPES[id]; } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/rpc/ChannelManager.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/rpc/ChannelManager.java index 73c8e65a1baf..c5ad10469298 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/rpc/ChannelManager.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/rpc/ChannelManager.java @@ -16,7 +16,7 @@ import com.alibaba.graphscope.groot.Utils; import com.alibaba.graphscope.groot.common.RoleType; import com.alibaba.graphscope.groot.common.config.*; -import com.alibaba.graphscope.groot.common.exception.NodeConnectException; +import com.alibaba.graphscope.groot.common.exception.NetworkFailureException; import io.grpc.ManagedChannel; import io.grpc.ManagedChannelBuilder; @@ -120,7 +120,7 @@ private ManagedChannel createChannel(RoleType role, int idx) { public ManagedChannel getChannel(RoleType role, int idx) { Map idToChannel = this.roleToChannels.get(role); if (idToChannel == null) { - throw new NodeConnectException("invalid role [" + role + "]"); + throw new NetworkFailureException("invalid role [" + role + "]"); } // to avoid thread competition if (idToChannel.get(idx) == null) { diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/AbstractAddTypePropertiesExecutor.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/AbstractAddTypePropertiesExecutor.java index 9c23cf27f271..27993726ceb0 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/AbstractAddTypePropertiesExecutor.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/AbstractAddTypePropertiesExecutor.java @@ -1,9 +1,9 @@ package com.alibaba.graphscope.groot.schema.ddl; +import com.alibaba.graphscope.groot.common.exception.DdlException; import com.alibaba.graphscope.groot.common.schema.wrapper.*; import com.alibaba.graphscope.groot.common.util.JSON; import com.alibaba.graphscope.groot.operation.Operation; -import com.alibaba.graphscope.groot.schema.request.DdlException; import com.alibaba.graphscope.proto.groot.TypeDefPb; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/AbstractCreateTypeExecutor.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/AbstractCreateTypeExecutor.java index a371392b87e3..20b7c22b1c42 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/AbstractCreateTypeExecutor.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/AbstractCreateTypeExecutor.java @@ -13,13 +13,13 @@ */ package com.alibaba.graphscope.groot.schema.ddl; +import com.alibaba.graphscope.groot.common.exception.DdlException; import com.alibaba.graphscope.groot.common.schema.wrapper.GraphDef; import com.alibaba.graphscope.groot.common.schema.wrapper.LabelId; import com.alibaba.graphscope.groot.common.schema.wrapper.PropertyDef; import com.alibaba.graphscope.groot.common.schema.wrapper.TypeDef; import com.alibaba.graphscope.groot.common.schema.wrapper.TypeEnum; import com.alibaba.graphscope.groot.operation.Operation; -import com.alibaba.graphscope.groot.schema.request.DdlException; import com.alibaba.graphscope.proto.groot.TypeDefPb; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/AbstractDdlExecutor.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/AbstractDdlExecutor.java index b3fd7f8ac809..4976ac286cf7 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/AbstractDdlExecutor.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/AbstractDdlExecutor.java @@ -13,9 +13,9 @@ */ package com.alibaba.graphscope.groot.schema.ddl; +import com.alibaba.graphscope.groot.common.exception.DdlException; import com.alibaba.graphscope.groot.common.schema.wrapper.GraphDef; import com.alibaba.graphscope.groot.common.schema.wrapper.PropertyDef; -import com.alibaba.graphscope.groot.schema.request.DdlException; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/AbstractDropTypeExecutor.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/AbstractDropTypeExecutor.java index 709bb0c9492b..7e8f6e90314a 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/AbstractDropTypeExecutor.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/AbstractDropTypeExecutor.java @@ -13,9 +13,9 @@ */ package com.alibaba.graphscope.groot.schema.ddl; +import com.alibaba.graphscope.groot.common.exception.DdlException; import com.alibaba.graphscope.groot.common.schema.wrapper.*; import com.alibaba.graphscope.groot.operation.Operation; -import com.alibaba.graphscope.groot.schema.request.DdlException; import com.alibaba.graphscope.proto.groot.TypeDefPb; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/AddEdgeKindExecutor.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/AddEdgeKindExecutor.java index 948ef2d9d44c..8301a68f059a 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/AddEdgeKindExecutor.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/AddEdgeKindExecutor.java @@ -13,12 +13,12 @@ */ package com.alibaba.graphscope.groot.schema.ddl; +import com.alibaba.graphscope.groot.common.exception.DdlException; import com.alibaba.graphscope.groot.common.schema.wrapper.EdgeKind; import com.alibaba.graphscope.groot.common.schema.wrapper.GraphDef; import com.alibaba.graphscope.groot.common.schema.wrapper.LabelId; import com.alibaba.graphscope.groot.operation.Operation; import com.alibaba.graphscope.groot.operation.ddl.AddEdgeKindOperation; -import com.alibaba.graphscope.groot.schema.request.DdlException; import com.alibaba.graphscope.proto.groot.EdgeKindPb; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/CommitDataLoadExecutor.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/CommitDataLoadExecutor.java index fab1b0c0f5b4..61af2acea91a 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/CommitDataLoadExecutor.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/CommitDataLoadExecutor.java @@ -13,6 +13,7 @@ */ package com.alibaba.graphscope.groot.schema.ddl; +import com.alibaba.graphscope.groot.common.exception.DdlException; import com.alibaba.graphscope.groot.common.schema.wrapper.EdgeKind; import com.alibaba.graphscope.groot.common.schema.wrapper.GraphDef; import com.alibaba.graphscope.groot.common.schema.wrapper.LabelId; @@ -21,7 +22,6 @@ import com.alibaba.graphscope.groot.common.util.DataLoadTarget; import com.alibaba.graphscope.groot.operation.Operation; import com.alibaba.graphscope.groot.operation.ddl.CommitDataLoadOperation; -import com.alibaba.graphscope.groot.schema.request.DdlException; import com.alibaba.graphscope.proto.groot.CommitDataLoadPb; import com.alibaba.graphscope.proto.groot.DataLoadTargetPb; import com.google.protobuf.ByteString; diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/DdlExecutors.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/DdlExecutors.java index 77d59c12bfe8..26d54c46142b 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/DdlExecutors.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/DdlExecutors.java @@ -13,6 +13,7 @@ */ package com.alibaba.graphscope.groot.schema.ddl; +import com.alibaba.graphscope.groot.common.exception.UnsupportedOperationException; import com.alibaba.graphscope.groot.common.schema.wrapper.GraphDef; import com.alibaba.graphscope.groot.operation.Operation; import com.alibaba.graphscope.groot.operation.OperationType; diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/PrepareDataLoadExecutor.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/PrepareDataLoadExecutor.java index 069817b12302..a9c7c745d9b5 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/PrepareDataLoadExecutor.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/PrepareDataLoadExecutor.java @@ -13,6 +13,7 @@ */ package com.alibaba.graphscope.groot.schema.ddl; +import com.alibaba.graphscope.groot.common.exception.DdlException; import com.alibaba.graphscope.groot.common.schema.wrapper.EdgeKind; import com.alibaba.graphscope.groot.common.schema.wrapper.GraphDef; import com.alibaba.graphscope.groot.common.schema.wrapper.LabelId; @@ -21,7 +22,6 @@ import com.alibaba.graphscope.groot.common.util.DataLoadTarget; import com.alibaba.graphscope.groot.operation.Operation; import com.alibaba.graphscope.groot.operation.ddl.PrepareDataLoadOperation; -import com.alibaba.graphscope.groot.schema.request.DdlException; import com.alibaba.graphscope.proto.groot.DataLoadTargetPb; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/RemoveEdgeKindExecutor.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/RemoveEdgeKindExecutor.java index c7b98257549f..e091b5c7b3db 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/RemoveEdgeKindExecutor.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/schema/ddl/RemoveEdgeKindExecutor.java @@ -13,12 +13,12 @@ */ package com.alibaba.graphscope.groot.schema.ddl; +import com.alibaba.graphscope.groot.common.exception.DdlException; import com.alibaba.graphscope.groot.common.schema.wrapper.EdgeKind; import com.alibaba.graphscope.groot.common.schema.wrapper.GraphDef; import com.alibaba.graphscope.groot.common.schema.wrapper.LabelId; import com.alibaba.graphscope.groot.operation.Operation; import com.alibaba.graphscope.groot.operation.ddl.RemoveEdgeKindOperation; -import com.alibaba.graphscope.groot.schema.request.DdlException; import com.alibaba.graphscope.proto.groot.EdgeKindPb; import com.google.protobuf.ByteString; import com.google.protobuf.InvalidProtocolBufferException; diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/KafkaProcessor.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/KafkaProcessor.java index 826f8ae3f020..8202e39f6a9f 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/KafkaProcessor.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/KafkaProcessor.java @@ -5,7 +5,8 @@ import com.alibaba.graphscope.groot.common.config.Configs; import com.alibaba.graphscope.groot.common.config.CoordinatorConfig; import com.alibaba.graphscope.groot.common.config.StoreConfig; -import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.groot.common.exception.IllegalStateException; +import com.alibaba.graphscope.groot.common.exception.InternalException; import com.alibaba.graphscope.groot.common.util.PartitionUtils; import com.alibaba.graphscope.groot.common.util.ThreadFactoryUtils; import com.alibaba.graphscope.groot.meta.FileMetaStore; @@ -79,7 +80,7 @@ public void start() { try { recover(); } catch (IOException e) { - throw new GrootException(e); + throw new InternalException(e); } this.persistOffsetsScheduler = @@ -156,7 +157,7 @@ public void recover() throws IOException { if (recoveredOffset != -1) { // if -1, then assume it's a fresh store try (LogReader ignored = logService.createReader(storeId, recoveredOffset + 1)) { } catch (Exception e) { - throw new IOException( + throw new IllegalStateException( "recovered queue [0] offset [" + recoveredOffset + "] is not available", e); } } @@ -193,7 +194,7 @@ public void pollBatches() { try { replayWAL(); } catch (IOException e) { - throw new RuntimeException(e); + throw new InternalException(e); } // -1 stands for poll from latest try (LogReader reader = logService.createReader(storeId, -1)) { @@ -204,7 +205,7 @@ public void pollBatches() { } } } catch (IOException e) { - throw new RuntimeException(e); + throw new InternalException(e); } } @@ -246,7 +247,7 @@ private void processRecord(ConsumerRecord record) { try { writerAgent.writeStore(builder.build()); } catch (InterruptedException e) { - throw new RuntimeException(e); + throw new InternalException(e); } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/SnapshotSortQueue.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/SnapshotSortQueue.java index bb3a48961f38..cb86eaf5ec11 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/SnapshotSortQueue.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/SnapshotSortQueue.java @@ -15,6 +15,7 @@ import com.alibaba.graphscope.groot.common.config.Configs; import com.alibaba.graphscope.groot.common.config.StoreConfig; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.groot.meta.MetaService; import com.alibaba.graphscope.groot.operation.StoreDataBatch; @@ -65,7 +66,7 @@ public SnapshotSortQueue(Configs configs, MetaService metaService) { public boolean offerQueue(int queueId, StoreDataBatch entry) throws InterruptedException { BlockingQueue innerQueue = this.innerQueues.get(queueId); if (innerQueue == null) { - throw new IllegalArgumentException("invalid queueId [" + queueId + "]"); + throw new InvalidArgumentException("invalid queueId [" + queueId + "]"); } boolean res = innerQueue.offer(entry, this.queueWaitMs, TimeUnit.MILLISECONDS); if (res) { diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/StoreService.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/StoreService.java index 42e8341f925b..1afa69a48fa8 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/StoreService.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/StoreService.java @@ -18,6 +18,8 @@ import com.alibaba.graphscope.groot.common.config.Configs; import com.alibaba.graphscope.groot.common.config.StoreConfig; import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.groot.common.exception.IllegalStateException; +import com.alibaba.graphscope.groot.common.exception.InternalException; import com.alibaba.graphscope.groot.common.util.ThreadFactoryUtils; import com.alibaba.graphscope.groot.common.util.Utils; import com.alibaba.graphscope.groot.meta.MetaService; @@ -101,7 +103,7 @@ public void start() throws IOException { GraphPartition partition = makeGraphPartition(this.storeConfigs, partitionId); this.idToPartition.put(partitionId, partition); } catch (IOException e) { - throw new GrootException(e); + throw new InternalException(e); } } initMetrics(); diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/external/ExternalStorage.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/external/ExternalStorage.java index 8ee588dbb970..a5ab2db00114 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/external/ExternalStorage.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/external/ExternalStorage.java @@ -1,5 +1,7 @@ package com.alibaba.graphscope.groot.store.external; +import com.alibaba.graphscope.groot.common.exception.*; + import org.apache.commons.codec.binary.Hex; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -34,7 +36,7 @@ public static ExternalStorage getStorage(String path, Map config case "volume": return new VolumeStorage(path, config); default: - throw new IllegalArgumentException( + throw new InvalidArgumentException( "external storage scheme [" + scheme + "] not supported"); } } @@ -60,7 +62,7 @@ public void downloadDataWithMove(String srcPath, String dstPath) throws IOExcept Files.move(Path.of(tmpPath), Path.of(dstPath), StandardCopyOption.REPLACE_EXISTING); } - public void downloadDataWithRetry(String srcPath, String dstPath) throws IOException { + public void downloadDataWithRetry(String srcPath, String dstPath) { int maxRetry = 5; for (int i = 0; i < maxRetry; ++i) { try { @@ -68,7 +70,7 @@ public void downloadDataWithRetry(String srcPath, String dstPath) throws IOExcep break; } catch (IOException e) { if (i == maxRetry - 1) { - throw e; + throw new ExternalStorageErrorException(e); } else { logger.error("Failed to download " + srcPath + ", retrying...", e); } @@ -89,7 +91,7 @@ public void downloadData(String srcPath, String dstPath) throws IOException { fis.read(chkData); fis.close(); } catch (FileNotFoundException e) { - throw new IOException(e); + throw new NotFoundException(e); } String[] chkArray = new String(chkData).split(","); if ("0".equals(chkArray[0])) { @@ -97,7 +99,7 @@ public void downloadData(String srcPath, String dstPath) throws IOException { return; } if (chkArray.length != 2) { - throw new IOException( + throw new InvalidDataException( "Checksum format error: content: [" + chkArray + "]; path: " + chkPath); } String chkMD5Value = chkArray[1]; @@ -106,7 +108,7 @@ public void downloadData(String srcPath, String dstPath) throws IOException { if (!chkMD5Value.equals(sstMD5Value)) { logger.error("Checksum failed for " + chkLocalPath + " versus " + dstPath); logger.error("Expect [" + chkMD5Value + "], got [" + sstMD5Value + "]"); - throw new IOException("CheckSum failed for " + srcPath); + throw new InvalidDataException("CheckSum failed for " + srcPath); } else { // The .chk file are now useless chkFile.delete(); @@ -125,9 +127,9 @@ public String getFileMD5(String fileName) throws IOException { } return new String(Hex.encodeHex(MD5.digest())); } catch (NoSuchAlgorithmException e) { - throw new IOException(e); + throw new InternalException(e); } catch (IOException e) { - throw e; + throw new InvalidArgumentException(e); } finally { if (fis != null) { fis.close(); diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/external/VolumeStorage.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/external/VolumeStorage.java index 4357ae442955..b32f90e7651f 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/external/VolumeStorage.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/external/VolumeStorage.java @@ -1,6 +1,7 @@ package com.alibaba.graphscope.groot.store.external; import com.alibaba.graphscope.groot.common.config.DataLoadConfig; +import com.alibaba.graphscope.groot.common.exception.ExternalStorageErrorException; import com.aliyun.odps.Odps; import com.aliyun.odps.account.AliyunAccount; import com.aliyun.odps.tunnel.TunnelException; @@ -56,7 +57,7 @@ public void downloadDataSimple(String srcPath, String dstPath) throws IOExceptio } } } catch (TunnelException e) { - throw new IOException(e); + throw new ExternalStorageErrorException(e); } long finish = System.currentTimeMillis(); long timeElapsed = finish - start; diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/jna/JnaGraphBackupEngine.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/jna/JnaGraphBackupEngine.java index 8dbf733598a2..9f769c65e55b 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/jna/JnaGraphBackupEngine.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/jna/JnaGraphBackupEngine.java @@ -13,6 +13,8 @@ */ package com.alibaba.graphscope.groot.store.jna; +import com.alibaba.graphscope.groot.common.exception.ExternalStorageErrorException; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.groot.store.backup.GraphPartitionBackup; import com.sun.jna.Pointer; @@ -53,7 +55,7 @@ public int createNewPartitionBackup() throws IOException { } byte[] data = jnaResponse.getData(); if (data == null || data.length != Integer.BYTES) { - throw new IOException( + throw new ExternalStorageErrorException( "fail to get new created backup id from jna response, partition [" + this.partitionId + "]"); @@ -69,14 +71,14 @@ public int createNewPartitionBackup() throws IOException { public void restoreFromPartitionBackup(int partitionBackupId, String PartitionRestorePath) throws IOException { if (PartitionRestorePath.equals(this.backupPath)) { - throw new IOException("restore path cannot be same with backup path"); + throw new InvalidArgumentException("restore path cannot be same with backup path"); } try (JnaResponse jnaResponse = GraphLibrary.INSTANCE.restoreFromBackup( this.bePointer, PartitionRestorePath, partitionBackupId)) { if (!jnaResponse.success()) { String errMsg = jnaResponse.getErrMsg(); - throw new IOException(errMsg); + throw new ExternalStorageErrorException(errMsg); } } } @@ -87,7 +89,7 @@ public void verifyPartitionBackup(int partitionBackupId) throws IOException { GraphLibrary.INSTANCE.verifyBackup(this.bePointer, partitionBackupId)) { if (!jnaResponse.success()) { String errMsg = jnaResponse.getErrMsg(); - throw new IOException(errMsg); + throw new ExternalStorageErrorException(errMsg); } } } @@ -100,7 +102,7 @@ public void partitionBackupGc(List readyPartitionBackupIds) throws IOEx try (JnaResponse jnaResponse = GraphLibrary.INSTANCE.getBackupList(this.bePointer)) { if (!jnaResponse.success()) { String errMsg = jnaResponse.getErrMsg(); - throw new IOException(errMsg); + throw new ExternalStorageErrorException(errMsg); } byte[] data = jnaResponse.getData(); if (data == null) { diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/jna/JnaGraphStore.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/jna/JnaGraphStore.java index 35c173d32bc5..554c0c0e1998 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/jna/JnaGraphStore.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/store/jna/JnaGraphStore.java @@ -16,6 +16,7 @@ import com.alibaba.graphscope.groot.common.config.CommonConfig; import com.alibaba.graphscope.groot.common.config.Configs; import com.alibaba.graphscope.groot.common.config.StoreConfig; +import com.alibaba.graphscope.groot.common.exception.ExternalStorageErrorException; import com.alibaba.graphscope.groot.operation.OperationBatch; import com.alibaba.graphscope.groot.store.GraphPartition; import com.alibaba.graphscope.groot.store.backup.GraphPartitionBackup; @@ -103,7 +104,7 @@ public boolean writeBatch(long snapshotId, OperationBatch operationBatch) throws this.pointer, snapshotId, dataBytes, dataBytes.length)) { if (!response.success()) { String errMsg = response.getErrMsg(); - throw new IOException(errMsg); + throw new ExternalStorageErrorException(errMsg); } return response.hasDdl(); } @@ -114,7 +115,7 @@ public GraphDefPb getGraphDefBlob() throws IOException { try (JnaResponse jnaResponse = GraphLibrary.INSTANCE.getGraphDefBlob(this.pointer)) { if (!jnaResponse.success()) { String errMsg = jnaResponse.getErrMsg(); - throw new IOException(errMsg); + throw new ExternalStorageErrorException(errMsg); } return GraphDefPb.parseFrom(jnaResponse.getData()); } @@ -125,7 +126,7 @@ public Statistics getGraphStatisticsBlob(long si) throws IOException { try (JnaResponse jnaResponse = GraphLibrary.INSTANCE.getGraphStatistics(this.pointer, si)) { if (!jnaResponse.success()) { String errMsg = jnaResponse.getErrMsg(); - throw new IOException(errMsg); + throw new ExternalStorageErrorException(errMsg); } return Statistics.parseFrom(jnaResponse.getData()); } @@ -156,7 +157,7 @@ public void garbageCollect(long snapshotId) throws IOException { try (JnaResponse response = GraphLibrary.INSTANCE.garbageCollectSnapshot(this.pointer, snapshotId)) { if (!response.success()) { - throw new IOException(response.getErrMsg()); + throw new ExternalStorageErrorException(response.getErrMsg()); } } } @@ -166,7 +167,7 @@ public void tryCatchUpWithPrimary() throws IOException { ensurePointer(); try (JnaResponse response = GraphLibrary.INSTANCE.tryCatchUpWithPrimary(this.pointer)) { if (!response.success()) { - throw new IOException(response.getErrMsg()); + throw new ExternalStorageErrorException(response.getErrMsg()); } } } @@ -176,7 +177,7 @@ public void reopenSecondary(long wait_sec) throws IOException { ensurePointer(); try (JnaResponse response = GraphLibrary.INSTANCE.reopenSecondary(this.pointer, wait_sec)) { if (!response.success()) { - throw new IOException(response.getErrMsg()); + throw new ExternalStorageErrorException(response.getErrMsg()); } } } @@ -186,14 +187,14 @@ public void compact() throws IOException { ensurePointer(); try (JnaResponse response = GraphLibrary.INSTANCE.compact(this.pointer)) { if (!response.success()) { - throw new IOException(response.getErrMsg()); + throw new ExternalStorageErrorException(response.getErrMsg()); } } } private void ensurePointer() throws IOException { if (this.pointer == null) { - throw new IOException("JNA pointer is null"); + throw new ExternalStorageErrorException("JNA pointer is null"); } } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/wal/kafka/KafkaLogReader.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/wal/kafka/KafkaLogReader.java index ac8b3dcf04c5..a0381a31759d 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/wal/kafka/KafkaLogReader.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/wal/kafka/KafkaLogReader.java @@ -15,6 +15,8 @@ */ package com.alibaba.graphscope.groot.wal.kafka; +import com.alibaba.graphscope.groot.common.exception.InternalException; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import com.alibaba.graphscope.groot.wal.LogEntry; import com.alibaba.graphscope.groot.wal.LogReader; import com.alibaba.graphscope.groot.wal.ReadLogEntry; @@ -63,7 +65,7 @@ public KafkaLogReader( offset = getOffset(client, partition, OffsetSpec.forTimestamp(timestamp)); } if (earliest > offset || offset > latest) { - throw new IllegalArgumentException( + throw new InvalidArgumentException( "invalid offset " + offset + ", hint: [" + earliest + ", " + latest + ")"); } consumer = new KafkaConsumer<>(kafkaConfigs, deSer, deSer); @@ -78,15 +80,14 @@ public KafkaLogReader( latest); } - private long getOffset(AdminClient client, TopicPartition partition, OffsetSpec spec) - throws IOException { + private long getOffset(AdminClient client, TopicPartition partition, OffsetSpec spec) { try { return client.listOffsets(Collections.singletonMap(partition, spec)) .partitionResult(partition) .get() .offset(); } catch (InterruptedException | ExecutionException e) { - throw new IOException(e); + throw new InternalException(e); } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/wal/kafka/KafkaLogService.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/wal/kafka/KafkaLogService.java index 2679ebbe626c..fe057d787b15 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/wal/kafka/KafkaLogService.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/wal/kafka/KafkaLogService.java @@ -16,7 +16,7 @@ import com.alibaba.graphscope.groot.common.config.CommonConfig; import com.alibaba.graphscope.groot.common.config.Configs; import com.alibaba.graphscope.groot.common.config.KafkaConfig; -import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.groot.common.exception.InternalException; import com.alibaba.graphscope.groot.wal.LogReader; import com.alibaba.graphscope.groot.wal.LogService; import com.alibaba.graphscope.groot.wal.LogWriter; @@ -68,7 +68,7 @@ public void init() { try { admin.createTopics(Collections.singleton(newTopic)).all().get(); } catch (InterruptedException | ExecutionException e) { - throw new GrootException("create topics [" + this.topic + "] failed", e); + throw new InternalException("create topics [" + this.topic + "] failed", e); } } @@ -78,7 +78,7 @@ public void destroy() { try { admin.deleteTopics(Arrays.asList(topic)).all().get(); } catch (InterruptedException | ExecutionException e) { - throw new GrootException("delete topics [" + this.topic + "] failed", e); + throw new InternalException("delete topics [" + this.topic + "] failed", e); } } @@ -88,7 +88,7 @@ public boolean initialized() { try { return admin.listTopics().names().get().contains(this.topic); } catch (InterruptedException | ExecutionException e) { - throw new GrootException("list topics failed", e); + throw new InternalException("list topics failed", e); } } @@ -134,7 +134,7 @@ public void deleteBeforeOffset(int queueId, long offset) throws IOException { try { result.all().get(); } catch (InterruptedException | ExecutionException e) { - throw new IOException(e); + throw new InternalException(e); } } @@ -166,6 +166,6 @@ private AdminClient createAdminWithRetry() throws InterruptedException { Thread.sleep(10000); } } - throw new RuntimeException("Create Kafka Client failed"); + throw new InternalException("Create Kafka Client failed"); } } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/wal/kafka/KafkaLogWriter.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/wal/kafka/KafkaLogWriter.java index 94666f87c898..979dc3e19e8d 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/wal/kafka/KafkaLogWriter.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/wal/kafka/KafkaLogWriter.java @@ -15,7 +15,7 @@ */ package com.alibaba.graphscope.groot.wal.kafka; -import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.groot.common.exception.InternalException; import com.alibaba.graphscope.groot.wal.LogEntry; import com.alibaba.graphscope.groot.wal.LogWriter; @@ -78,7 +78,7 @@ public long waitFuture(Future future) { recordMetadata = future.get(); } catch (InterruptedException | ExecutionException e) { logger.error("append kafka failed", e); - throw new GrootException(e); + throw new InternalException(e); } return recordMetadata.offset(); } diff --git a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/wal/kafka/LogEntryDeserializer.java b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/wal/kafka/LogEntryDeserializer.java index 4b0d714e1e08..e337b594cc16 100644 --- a/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/wal/kafka/LogEntryDeserializer.java +++ b/interactive_engine/groot-module/src/main/java/com/alibaba/graphscope/groot/wal/kafka/LogEntryDeserializer.java @@ -15,7 +15,7 @@ */ package com.alibaba.graphscope.groot.wal.kafka; -import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.groot.common.exception.InternalException; import com.alibaba.graphscope.groot.wal.LogEntry; import com.alibaba.graphscope.proto.groot.LogEntryPb; import com.google.protobuf.InvalidProtocolBufferException; @@ -36,7 +36,7 @@ public LogEntry deserialize(String topic, byte[] data) { try { return LogEntry.parseProto(LogEntryPb.parseFrom(data)); } catch (InvalidProtocolBufferException e) { - throw new GrootException(e); + throw new InternalException(e); } } diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/Coordinator.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/Coordinator.java index 0c9ee6b9d47c..4f2cdad2a345 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/Coordinator.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/Coordinator.java @@ -18,7 +18,7 @@ import com.alibaba.graphscope.groot.common.config.CommonConfig; import com.alibaba.graphscope.groot.common.config.Configs; import com.alibaba.graphscope.groot.common.config.CoordinatorConfig; -import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.groot.common.exception.InternalException; import com.alibaba.graphscope.groot.coordinator.*; import com.alibaba.graphscope.groot.coordinator.IngestorWriteClient; import com.alibaba.graphscope.groot.coordinator.backup.BackupManager; @@ -162,7 +162,7 @@ public void start() { try { this.rpcServer.start(); } catch (IOException e) { - throw new GrootException(e); + throw new InternalException(e); } this.discovery.start(); this.channelManager.start(); diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/Frontend.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/Frontend.java index be77141b1fac..bfaf456df732 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/Frontend.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/Frontend.java @@ -18,7 +18,7 @@ import com.alibaba.graphscope.groot.common.config.CommonConfig; import com.alibaba.graphscope.groot.common.config.Configs; import com.alibaba.graphscope.groot.common.config.FrontendConfig; -import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.groot.common.exception.InternalException; import com.alibaba.graphscope.groot.common.util.RpcUtils; import com.alibaba.graphscope.groot.discovery.FileDiscovery; import com.alibaba.graphscope.groot.discovery.LocalNodeProvider; @@ -169,7 +169,7 @@ public void start() { try { this.rpcServer.start(); } catch (IOException e) { - throw new GrootException(e); + throw new InternalException(e); } this.discovery.start(); this.channelManager.start(); @@ -179,14 +179,14 @@ public void start() { Thread.sleep(1000); logger.info("Waiting for schema ready..."); } catch (InterruptedException e) { - throw new GrootException(e); + throw new InternalException(e); } } this.graphService.start(); try { this.serviceServer.start(); } catch (IOException e) { - throw new GrootException(e); + throw new InternalException(e); } } diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/GrootGraph.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/GrootGraph.java index 516ebd4b0331..08e9939e5371 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/GrootGraph.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/GrootGraph.java @@ -18,7 +18,8 @@ import com.alibaba.graphscope.groot.Utils; import com.alibaba.graphscope.groot.common.RoleType; import com.alibaba.graphscope.groot.common.config.*; -import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.groot.common.exception.InternalException; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import io.opentelemetry.api.OpenTelemetry; @@ -42,7 +43,7 @@ public static void main(String[] args) throws Exception { try { node = new MaxNode(conf); } catch (Exception e) { - throw new GrootException(e); + throw new InternalException(e); } } else { String roleName = args[0]; @@ -58,7 +59,7 @@ public static void main(String[] args) throws Exception { node = new Coordinator(conf); break; default: - throw new IllegalArgumentException("invalid roleType [" + roleType + "]"); + throw new InvalidArgumentException("invalid roleType [" + roleType + "]"); } boolean writeHAEnabled = CommonConfig.WRITE_HA_ENABLED.get(conf); diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/MaxNode.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/MaxNode.java index 02d9fbd05f6f..8d635aab1eab 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/MaxNode.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/MaxNode.java @@ -15,7 +15,7 @@ import com.alibaba.graphscope.groot.common.RoleType; import com.alibaba.graphscope.groot.common.config.*; -import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.groot.common.exception.InternalException; import com.google.common.annotations.VisibleForTesting; import com.salesforce.kafka.test.KafkaTestCluster; @@ -113,7 +113,7 @@ public void start() { try { startThread.join(); } catch (InterruptedException e) { - throw new GrootException(e); + throw new InternalException(e); } } logger.info("MaxNode started"); diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/NodeLauncher.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/NodeLauncher.java index 479452baafa6..18f26d405f2d 100755 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/NodeLauncher.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/NodeLauncher.java @@ -13,7 +13,7 @@ */ package com.alibaba.graphscope.groot.servers; -import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.groot.common.exception.InternalException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -59,7 +59,7 @@ public void start() { this.node.start(); } catch (Exception e) { logger.error("start node failed", e); - throw new GrootException(e); + throw new InternalException(e); } this.keepAliveThread.start(); } diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/Store.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/Store.java index 4a6e5b52e46d..ae86914a8aac 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/Store.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/Store.java @@ -15,7 +15,7 @@ import com.alibaba.graphscope.groot.common.RoleType; import com.alibaba.graphscope.groot.common.config.Configs; -import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.groot.common.exception.InternalException; import com.alibaba.graphscope.groot.discovery.*; import com.alibaba.graphscope.groot.meta.DefaultMetaService; import com.alibaba.graphscope.groot.meta.MetaService; @@ -91,14 +91,14 @@ public void start() { try { this.storeService.start(); } catch (IOException e) { - throw new GrootException(e); + throw new InternalException(e); } this.writerAgent.start(); this.backupAgent.start(); try { this.rpcServer.start(); } catch (IOException e) { - throw new GrootException(e); + throw new InternalException(e); } this.discovery.start(); this.channelManager.start(); diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/FrontendQueryManager.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/FrontendQueryManager.java index 0330544d8c39..59d42c54d46e 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/FrontendQueryManager.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/FrontendQueryManager.java @@ -19,6 +19,7 @@ import com.alibaba.graphscope.common.ir.meta.IrMeta; import com.alibaba.graphscope.common.ir.meta.fetcher.IrMetaFetcher; import com.alibaba.graphscope.common.manager.IrMetaQueryCallback; +import com.alibaba.graphscope.groot.common.exception.InternalException; import com.alibaba.graphscope.groot.frontend.SnapshotUpdateClient; import com.alibaba.graphscope.groot.rpc.RoleClients; import com.google.common.util.concurrent.ThreadFactoryBuilder; @@ -87,7 +88,7 @@ public synchronized IrMeta beforeExec() { queryQueue.put(status); return irMeta; } catch (InterruptedException e) { - throw new RuntimeException(e); + throw new InternalException(e); } } diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GaiaEngine.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GaiaEngine.java index db765295cbd0..e7043655e697 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GaiaEngine.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/GaiaEngine.java @@ -19,7 +19,7 @@ import com.alibaba.graphscope.groot.common.RoleType; import com.alibaba.graphscope.groot.common.config.CommonConfig; import com.alibaba.graphscope.groot.common.config.Configs; -import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.groot.common.exception.GaiaInternalException; import com.alibaba.graphscope.groot.discovery.*; import com.alibaba.graphscope.groot.servers.jna.GaiaLibrary; import com.alibaba.graphscope.groot.servers.jna.GaiaPortsResponse; @@ -82,7 +82,7 @@ public void updatePartitionRouting(int partitionId, int serverId) { public void start() { try (GaiaPortsResponse gaiaPortsResponse = GaiaLibrary.INSTANCE.startEngine(this.pointer)) { if (!gaiaPortsResponse.success) { - throw new GrootException(gaiaPortsResponse.errMsg); + throw new GaiaInternalException(gaiaPortsResponse.errMsg); } engineNodeProvider.apply(gaiaPortsResponse.enginePort); rpcNodeProvider.apply(gaiaPortsResponse.rpcPort); diff --git a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java index 61f5835bf556..a8a3da0de562 100644 --- a/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java +++ b/interactive_engine/groot-server/src/main/java/com/alibaba/graphscope/groot/servers/ir/IrServiceProducer.java @@ -26,6 +26,7 @@ import com.alibaba.graphscope.groot.common.RoleType; import com.alibaba.graphscope.groot.common.config.CommonConfig; import com.alibaba.graphscope.groot.common.config.Configs; +import com.alibaba.graphscope.groot.common.exception.InternalException; import com.alibaba.graphscope.groot.common.schema.api.SchemaFetcher; import com.alibaba.graphscope.groot.discovery.DiscoveryFactory; import com.alibaba.graphscope.groot.frontend.SnapshotUpdateClient; @@ -83,7 +84,7 @@ public void start() { this.graphServer.start(); queryManager.start(); } catch (Exception e) { - throw new RuntimeException(e); + throw new InternalException(e); } } @@ -93,7 +94,7 @@ public void stop() { this.graphServer.close(); // graphServer is always not null queryManager.stop(); } catch (Exception e) { - throw new RuntimeException(e); + throw new InternalException(e); } } }; diff --git a/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/common/client/RoleClientsTest.java b/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/common/client/RoleClientsTest.java index cd26eabb581e..c750afebf461 100644 --- a/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/common/client/RoleClientsTest.java +++ b/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/common/client/RoleClientsTest.java @@ -18,7 +18,7 @@ import com.alibaba.graphscope.groot.common.RoleType; import com.alibaba.graphscope.groot.common.config.CommonConfig; import com.alibaba.graphscope.groot.common.config.Configs; -import com.alibaba.graphscope.groot.common.exception.NodeConnectException; +import com.alibaba.graphscope.groot.common.exception.NetworkFailureException; import com.alibaba.graphscope.groot.rpc.ChannelManager; import com.alibaba.graphscope.groot.rpc.RoleClients; import com.alibaba.graphscope.groot.rpc.RpcChannel; @@ -41,7 +41,7 @@ void testRoleClients() { new RoleClients<>(channelManager, RoleType.STORE, MockRoleClient::new); channelManager.start(); assertNotNull(clients.getClient(0)); - assertThrows(NodeConnectException.class, () -> clients.getClient(1)); + assertThrows(NetworkFailureException.class, () -> clients.getClient(1)); channelManager.stop(); } diff --git a/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/common/rpc/ChannelManagerTest.java b/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/common/rpc/ChannelManagerTest.java index ff7b4ff6b588..d2612a960ed3 100644 --- a/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/common/rpc/ChannelManagerTest.java +++ b/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/common/rpc/ChannelManagerTest.java @@ -16,7 +16,7 @@ import com.alibaba.graphscope.groot.common.RoleType; import com.alibaba.graphscope.groot.common.config.CommonConfig; import com.alibaba.graphscope.groot.common.config.Configs; -import com.alibaba.graphscope.groot.common.exception.NodeConnectException; +import com.alibaba.graphscope.groot.common.exception.NetworkFailureException; import com.alibaba.graphscope.groot.rpc.ChannelManager; import org.junit.jupiter.api.Assertions; @@ -36,7 +36,7 @@ void testChannelManager() { channelManager.start(); Assertions.assertNotNull(channelManager.getChannel(RoleType.STORE, 0)); Assertions.assertThrows( - NodeConnectException.class, () -> channelManager.getChannel(RoleType.STORE, 1)); + NetworkFailureException.class, () -> channelManager.getChannel(RoleType.STORE, 1)); channelManager.stop(); } } diff --git a/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/coordinator/CoordinatorRpcTest.java b/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/coordinator/CoordinatorRpcTest.java index 5ce6946ada60..d2925ead3c58 100644 --- a/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/coordinator/CoordinatorRpcTest.java +++ b/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/coordinator/CoordinatorRpcTest.java @@ -17,6 +17,7 @@ import static org.mockito.Mockito.*; import com.alibaba.graphscope.groot.CompletionCallback; +import com.alibaba.graphscope.groot.common.exception.DdlException; import com.alibaba.graphscope.groot.common.schema.wrapper.GraphDef; import com.alibaba.graphscope.groot.common.util.BackupInfo; import com.alibaba.graphscope.groot.coordinator.*; @@ -26,7 +27,6 @@ import com.alibaba.graphscope.groot.coordinator.backup.BackupService; import com.alibaba.graphscope.groot.coordinator.backup.StoreBackupClient; import com.alibaba.graphscope.groot.rpc.RoleClients; -import com.alibaba.graphscope.groot.schema.request.DdlException; import com.alibaba.graphscope.groot.store.backup.StoreBackupId; import com.alibaba.graphscope.proto.groot.*; diff --git a/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/gremlin/GrootGraph.java b/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/gremlin/GrootGraph.java index ed827949005b..dd52b2abdc00 100644 --- a/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/gremlin/GrootGraph.java +++ b/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/gremlin/GrootGraph.java @@ -18,7 +18,8 @@ import com.alibaba.graphscope.groot.common.config.CommonConfig; import com.alibaba.graphscope.groot.common.config.Configs; import com.alibaba.graphscope.groot.common.config.FrontendConfig; -import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; +import com.alibaba.graphscope.groot.common.exception.UnsupportedOperationException; import com.alibaba.graphscope.groot.sdk.GrootClient; import com.alibaba.graphscope.groot.sdk.schema.Edge; import com.alibaba.graphscope.groot.sdk.schema.Vertex; @@ -78,7 +79,7 @@ public GrootGraph(Configs configs) { this.remoteConnection = DriverRemoteConnection.using(cluster); } catch (Throwable e) { this.closeGraph(); - throw new GrootException(e); + throw new InvalidArgumentException(e); } } diff --git a/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/gremlin/GrootGraphProvider.java b/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/gremlin/GrootGraphProvider.java index eb23bffac8dc..17636917cc8c 100644 --- a/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/gremlin/GrootGraphProvider.java +++ b/interactive_engine/groot-server/src/test/java/com/alibaba/graphscope/groot/tests/gremlin/GrootGraphProvider.java @@ -14,7 +14,7 @@ package com.alibaba.graphscope.groot.tests.gremlin; import com.alibaba.graphscope.groot.common.config.StoreConfig; -import com.alibaba.graphscope.groot.common.exception.GrootException; +import com.alibaba.graphscope.groot.common.exception.InvalidArgumentException; import org.apache.commons.configuration2.Configuration; import org.apache.tinkerpop.gremlin.AbstractGraphProvider; @@ -51,7 +51,7 @@ public Map getBaseConfiguration( .getResourceAsStream("gremlin-test.config")) { properties.load(ins); } catch (IOException e) { - throw new RuntimeException(e); + throw new InvalidArgumentException(e); } properties.put(Graph.GRAPH, GrootGraph.class.getName()); if (this.storeDataPath == null) { @@ -80,7 +80,7 @@ public void loadGraphData( loadedGraphs.add(graphData); } catch (URISyntaxException | IOException e) { logger.error("load schema failed", e); - throw new GrootException(e); + throw new InvalidArgumentException(e); } catch (InterruptedException e) { logger.error("load data failed", e); } diff --git a/proto/error/analytical.proto b/proto/error/analytical.proto new file mode 100644 index 000000000000..7e751b83b6d1 --- /dev/null +++ b/proto/error/analytical.proto @@ -0,0 +1,80 @@ +/** + * + * The file proto/attr_value.proto is referred and derived from project + * tensorflow, + * + * https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/attr_value.proto + * + * which has the following license: + * + * + Copyright 2015 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ============================================================================== + */ +syntax = "proto3"; + +package gs.rpc; + +// component-02: Analytical Engine +enum Code { + // Not an error; returned on success. + OK = 0; + CANCELLED = 1; + // Arguments of operations is invalid / in a wrong format. + INVALID_ARGUMENT = 2; + // Timeout, used when an operation fail to return result in an specific time. + TIMEOUT = 3; + // Required resources cannot be found. + NOT_FOUND = 4; + // File or resources already existed. + ALREADY_EXISTS = 5; + RESOURCE_EXHAUSTED = 6; + // Functionality not implemented yet + UNIMPLEMENTED = 7; + // Client doesn't have the permission. + PERMISSION_DENIED = 8; + // Compile graph frame or app frame failed. + COMPILATION_FAILURE = 9; + PORT_IN_USE = 10; + UNSUPPORTED_OPERATION = 11; + ILLEGAL_STATE = 12; + // Network is unreachable + NETWORK_FAILURE = 13; + + // kOk = 0; + // kIOError = 100; + IO_ERROR = 100; + // kArrowError = 101; + ARROW_ERROR = 101; + // kVineyardError = 102; + VINEYARD_ERROR = 102; + // kUnspecificError = 103; + UNSPECIFIC_ERROR = 103; + // kDistributedError = 104; + DISTRIBUTED_ERROR = 104; + // kNetworkError = 105; + // kCommandError = 105; + COMMAND_ERROR = 105; + // kDataTypeError = 106; + DATA_TYPE_ERROR = 106; + // kIllegalStateError = 108; + // kInvalidValueError = 109; + // kInvalidOperationError = 107; + INVALID_OPERATION = 107; + // kUnsupportedOperationError = 111; + // kUnimplementedMethod = 112; + // kGraphArError = 108; + GRAPHAR_ERROR = 108; +} diff --git a/proto/error/coordinator.proto b/proto/error/coordinator.proto new file mode 100644 index 000000000000..8bf58db11a44 --- /dev/null +++ b/proto/error/coordinator.proto @@ -0,0 +1,98 @@ +/** + * + * The file proto/attr_value.proto is referred and derived from project + * tensorflow, + * + * https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/attr_value.proto + * + * which has the following license: + * + * + Copyright 2015 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ============================================================================== + */ +syntax = "proto3"; +package gs.rpc; + +// component-01: Coordinator +enum Code { + // Not an error; returned on success. + OK = 0; + CANCELLED = 1; + // Arguments of operations is invalid / in a wrong format. + INVALID_ARGUMENT = 2; + // Timeout, used when an operation fail to return result in an specific time. + TIMEOUT = 3; + // Required resources cannot be found. + NOT_FOUND = 4; + // File or resources already existed. + ALREADY_EXISTS = 5; + RESOURCE_EXHAUSTED = 6; + // Functionality not implemented yet + UNIMPLEMENTED = 7; + // Client doesn't have the permission. + PERMISSION_DENIED = 8; + // Compile graph frame or app frame failed. + COMPILATION_FAILURE = 9; + PORT_IN_USE = 10; + UNSUPPORTED_OPERATION = 11; + ILLEGAL_STATE = 12; + // Network is unreachable + NETWORK_FAILURE = 13; + + // Connection error with client. + // e.g. multiple client connect to coordinator at the same time or + // failed to launch engine locally. + CONNECTION_ERROR = 100; + + // Error raised by vineyard. + VINEYARD_ERROR = 101; + + // Error raised when running UDF application + UDF_INTERNAL_ERROR = 102; + + // Error reported by the k8s cluster + K8S_ERROR = 103; + + // Error raised when executing gremlin query + GREMLIN_QUERY_ERROR = 104; + + // Retry time exceed the defined maximum threshold + RETRIES_EXCEEDED_ERROR = 105; + + // Errors raised by analytical engine + ANALYTICAL_ENGINE_INTERNAL_ERROR = 106; + + // Errors raised by interactive engine + INTERACTIVE_ENGINE_INTERNAL_ERROR = 107; + + // Errors raised by learning engine + LEARNING_ENGINE_INTERNAL_ERROR = 108; + + // Errors raised by coordinator + COORDINATOR_INTERNAL_ERROR = 109; + + // Errors raised by analytical engine + // Results of workers not consistent + WORKER_RESULTS_INCONSISTENT_ERROR = 110; + + // Unknown error. + UNKNOWN_ERROR = 111; + + // Fatal error. + // Means the service is not recoverable. + // e.g. engine socket closed in grpc connection. Usually means engine has crashed. + FATAL_ERROR = 112; +} diff --git a/proto/error/error.proto b/proto/error/error.proto new file mode 100644 index 000000000000..0e3a84b4b683 --- /dev/null +++ b/proto/error/error.proto @@ -0,0 +1,36 @@ +/** + * + * The file proto/attr_value.proto is referred and derived from project + * tensorflow, + * + * https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/attr_value.proto + * + * which has the following license: + * + * + Copyright 2015 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ============================================================================== + */ +syntax = "proto3"; + +package gs.rpc; + +option java_package = "com.alibaba.graphscope.proto"; +option java_multiple_files = true; + +message Status { + int32 code = 1; + string message = 2; +} diff --git a/proto/error/frontend.proto b/proto/error/frontend.proto new file mode 100644 index 000000000000..474c4d9584cf --- /dev/null +++ b/proto/error/frontend.proto @@ -0,0 +1,95 @@ +/** + * + * The file proto/attr_value.proto is referred and derived from project + * tensorflow, + * + * https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/attr_value.proto + * + * which has the following license: + * + * + Copyright 2015 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ============================================================================== + */ +syntax = "proto3"; + +option java_package = "com.alibaba.graphscope.proto"; +option java_multiple_files = true; + +// component-03: GIE Frontend +enum Code { + // Not an error; returned on success. + OK = 0; + CANCELLED = 1; + // Arguments of operations is invalid / in a wrong format. + INVALID_ARGUMENT = 2; + // Timeout, used when an operation fail to return result in an specific time. + TIMEOUT = 3; + // Required resources cannot be found. + NOT_FOUND = 4; + // File or resources already existed. + ALREADY_EXISTS = 5; + RESOURCE_EXHAUSTED = 6; + // Functionality not implemented yet + UNIMPLEMENTED = 7; + // Client doesn't have the permission. + PERMISSION_DENIED = 8; + // Compile graph frame or app frame failed. + COMPILATION_FAILURE = 9; + PORT_IN_USE = 10; + UNSUPPORTED_OPERATION = 11; + ILLEGAL_STATE = 12; + // Network is unreachable + NETWORK_FAILURE = 13; + + // invalid gremlin syntax thrown by antlr4 + // GremlinSyntaxParseError = 100; + GREMLIN_INVALID_SYNTAX = 100; + // invalid cypher syntax thrown by antlr4 + // CypherSyntaxParseError = 101; + CYPHER_INVALID_SYNTAX = 101; + // query given label is not found in schema + // LabelNotFoundError = 102; + LABEL_NOT_FOUND = 102; + // query given property name is not found in schema + // PropertyNotFoundError = 103; + PROPERTY_NOT_FOUND = 103; + // tag not exist in current query contex + // TagNotFoundError = 104; + TAG_NOT_FOUND = 104; + // targets two types of errors: + // 1. invalid algebraic type, such as "cannot apply + to string arguments" + // 2. invalid graph patterns based on the given schema + // TypeInferenceError = 105; + TYPE_INFERENCE_FAILED = 105; + // error thrown in logical plan optimization + // LogicalPlanBuildError = 106; + LOGICAL_PLAN_BUILD_FAILED = 106; + // error thrown in physical plan build + // PhysicalPlanBuildError = 107; + PHYSICAL_PLAN_BUILD_FAILED = 107; + // error thrown when parsing gremlin results + // GremlinResultParseError = 108; + GREMLIN_INVALID_RESULT = 108; + // error thrown then parsing cypher results + // CypherResultParseError = 109; + CYPHER_INVALID_RESULT = 109; + // engine is not available to process the query, e.g. engine is down + // EngineUnavailableError = 110; + ENGINE_UNAVAILABLE = 110; + // query execution time exceeds the limit + // QueryExecutionTimeoutError = 111; + QUERY_EXECUTION_TIMEOUT = 111; +} diff --git a/proto/error/insight.proto b/proto/error/insight.proto new file mode 100644 index 000000000000..8bb83c7360d9 --- /dev/null +++ b/proto/error/insight.proto @@ -0,0 +1,123 @@ +/** + * + * The file proto/attr_value.proto is referred and derived from project + * tensorflow, + * + * https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/attr_value.proto + * + * which has the following license: + * + * + Copyright 2015 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ============================================================================== + */ +syntax = "proto3"; + +option java_package = "com.alibaba.graphscope.proto"; +option java_multiple_files = true; + +// component-04: GIE Insight Server (vineyard + groot) +enum Code { + // Not an error; returned on success. + OK = 0; + CANCELLED = 1; + // Arguments of operations is invalid / in a wrong format. + INVALID_ARGUMENT = 2; + // Timeout, used when an operation fail to return result in an specific time. + TIMEOUT = 3; + // Required resources cannot be found. + NOT_FOUND = 4; + // File or resources already existed. + ALREADY_EXISTS = 5; + RESOURCE_EXHAUSTED = 6; + // Functionality not implemented yet + UNIMPLEMENTED = 7; + // Client doesn't have the permission. + PERMISSION_DENIED = 8; + // Compile graph frame or app frame failed. + COMPILATION_FAILURE = 9; + PORT_IN_USE = 10; + UNSUPPORTED_OPERATION = 11; + ILLEGAL_STATE = 12; + // Network is unreachable + NETWORK_FAILURE = 13; + + // try to get a property in ValueType1, but the it's real type is ValueType2 and it's not ValueType1 and + // compatible to ValueType1 + // ValueTypeMismatch = 100; + VALUE_TYPE_MISMATCH = 100; + // get lock failed + // LockFailed = 101; + LOCK_FAILED = 101; + // too many data of old versions in graph store, maybe something error with garbage collection + // usuage trigger when modifing the schema + // TooManyVersions = 102; + TOO_MANY_VERSIONS = 102; + // some fatal bug in graph store + // GraphStoreBug = 103; + BACKUP_FAILED = 103; + // user's operation is invalid, like: create a type that already exists + // InvalidOperation = 104; + INVALID_OPERATION = 104; + // when try to insert data, the type exists in storage but isn't visible at that moment + // DataNotExists = 105; + DATA_NOT_EXISTS = 105; + // error in external storage like rocksdb + // ExternalStorageError = 106; + EXTERNAL_STORAGE_ERROR = 106; + // decode property from bytes failed, maybe caused by corrupted binary data in storage, + // or by using the wrong decoder, + // DecodeError = 107; + INVALID_DATA = 107; + // operations or features is not supported, e.g. delete from a secondary instance + // NotSupported = 108; + + // PropertyAlreadyExist = 109; + TYPE_NOT_FOUND = 108; + PROPERTY_NOT_FOUND = 109; + // TypeAlreadyExist = 111; + // RelationShipAlreadyExist = 113; + // RelationShipNotExist = 114; + + // client-frontend related + // RealtimeWriteFailed = 110; + REALTIME_WRITE_FAILED = 110; + // ServiceNotReady = 111; + SERVICE_NOT_READY = 111; + // QueueReject = 112; + QUEUE_REJECT = 112; + + // query related + // QueryFailed = 113; + QUERY_FAILED = 113; + // QueryCancelled = 114; + // QUERY_CANCELLED = 114; + + // IllegalSchema = 115; + ILLEGAL_SCHEMA = 115; + + // e.g. delete property + // UnSupportedMetaOperation = 115; + // e.g. list, map, set + // DataTypeNotValid = 109; + INVALID_DATA_TYPE = 116; + + INTERNAL = 117; + + GAIA_INTERNAL_ERROR = 118; + + DDL_ERROR = 119; + +} diff --git a/proto/error/interactive.proto b/proto/error/interactive.proto new file mode 100644 index 000000000000..99938352601b --- /dev/null +++ b/proto/error/interactive.proto @@ -0,0 +1,90 @@ +/** + * + * The file proto/attr_value.proto is referred and derived from project + * tensorflow, + * + * https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/attr_value.proto + * + * which has the following license: + * + * + Copyright 2015 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ============================================================================== + */ +syntax = "proto3"; + +package gs.rpc.interactive; + + +// component-05: GIE Interactive Server (flex) +enum Code { + // Not an error; returned on success. + OK = 0; + CANCELLED = 1; + // Arguments of operations is invalid / in a wrong format. + INVALID_ARGUMENT = 2; + // Timeout, used when an operation fail to return result in an specific time. + TIMEOUT = 3; + // Required resources cannot be found. + NOT_FOUND = 4; + // File or resources already existed. + ALREADY_EXISTS = 5; + RESOURCE_EXHAUSTED = 6; + // Functionality not implemented yet + UNIMPLEMENTED = 7; + // Client doesn't have the permission. + PERMISSION_DENIED = 8; + // Compile graph frame or app frame failed. + COMPILATION_FAILURE = 9; + PORT_IN_USE = 10; + UNSUPPORTED_OPERATION = 11; + ILLEGAL_STATE = 12; + // Network is unreachable + NETWORK_FAILURE = 13; + + // InValidArgument = 100; + // UnsupportedOperator = 101; + // AlreadyExists = 102; + // NotExists = 103; + // CodegenError = 100; + CODEGEN_ERROR = 100; + // UninitializedStatus = 101; + + // InvalidSchema = 101; + INVALID_SCHEMA = 101; + // PermissionError = 107; + // IllegalOperation = 102; + ILLEGAL_OPERATION = 102; + // InternalError = 103; + INTERNAL_ERROR = 103; + // InvalidImportFile = 104; + INVALID_IMPORT_FILE = 104; + // IOError = 105; + IO_ERROR = 105; + // NotFound = 112; + // QueryFailed = 106; + QUERY_FAILED = 106; + // ReopenError = 107; + REOPEN_ERROR = 107; + // ErrorOpenMeta = 108; + ERROR_OPEN_META = 108; + // SQlExecutionError = 109; + SQL_EXECUTION_ERROR = 109; + // SqlBindingError = 110; + SQL_BINDING_ERROR = 110; + // Unimplemented = 118; + // AlreadyLocked = 111; + ALREADY_LOCKED = 111; +} diff --git a/proto/error/learning.proto b/proto/error/learning.proto new file mode 100644 index 000000000000..efe6916ce25d --- /dev/null +++ b/proto/error/learning.proto @@ -0,0 +1,56 @@ + +/** + * + * The file proto/attr_value.proto is referred and derived from project + * tensorflow, + * + * https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/framework/attr_value.proto + * + * which has the following license: + * + * + Copyright 2015 The TensorFlow Authors. All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + ============================================================================== + */ +syntax = "proto3"; + +package gs.rpc.learning; + +// component-06: Learning Engine +enum Code { + // Not an error; returned on success. + OK = 0; + CANCELLED = 1; + // Arguments of operations is invalid / in a wrong format. + INVALID_ARGUMENT = 2; + // Timeout, used when an operation fail to return result in an specific time. + TIMEOUT = 3; + // Required resources cannot be found. + NOT_FOUND = 4; + // File or resources already existed. + ALREADY_EXISTS = 5; + RESOURCE_EXHAUSTED = 6; + // Functionality not implemented yet + UNIMPLEMENTED = 7; + // Client doesn't have the permission. + PERMISSION_DENIED = 8; + // Compile graph frame or app frame failed. + COMPILATION_FAILURE = 9; + PORT_IN_USE = 10; + UNSUPPORTED_OPERATION = 11; + ILLEGAL_STATE = 12; + // Network is unreachable + NETWORK_FAILURE = 13; +} diff --git a/proto/error_codes.proto b/proto/error_codes.proto deleted file mode 100644 index c2e582dce8a2..000000000000 --- a/proto/error_codes.proto +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// - -syntax = "proto3"; - -package gs.rpc; - -enum Code { - // Not an error; returned on success. - OK = 0; - - // Timeout, used when an operation fail to return result in an specific time. - TIMEOUT_ERROR = 1; - - // Required resources cannot be found. - NOT_FOUND_ERROR = 2; - - // Connection error with client. - // e.g. multiple client connect to coordinator at the same time or - // failed to launch engine locally. - CONNECTION_ERROR = 3; - - // Error raised by vineyard. - VINEYARD_ERROR = 4; - - // Compile graph frame or app frame failed. - COMPILATION_ERROR = 5; - - // File or resources already existed. - ALREADY_EXISTS_ERROR = 6; - - // Error raised when running UDF application - UDF_INTERNAL_ERROR = 7; - - // Resources temporary unavaiable. - UNAVAILABLE_ERROR = 8; - - // Arguments of operations is invalid / in a wrong format. - INVALID_ARGUMENT_ERROR = 9; - - // Client doesn't have the permission. - PERMISSION_DENIED_ERROR = 10; - - // Network is unreachable - NETWORK_ERROR = 11; - - // Error reported by the k8s cluster - K8S_ERROR = 12; - - // Functionality not implemented yet - UNIMPLEMENTED_ERROR = 13; - - // Error raised when executing gremlin query - GREMLIN_QUERY_ERROR = 14; - - // Retry time exceed the defined maximum threshold - RETRIES_EXCEEDED_ERROR = 15; - - // Errors raised by analytical engine - ANALYTICAL_ENGINE_INTERNAL_ERROR = 30; - - // Errors raised by interactive engine - INTERACTIVE_ENGINE_INTERNAL_ERROR = 31; - - // Errors raised by learning engine - LEARNING_ENGINE_INTERNAL_ERROR = 32; - - // Errors raised by coordinator - COORDINATOR_INTERNAL_ERROR = 33; - - // Errors raised by analytical engine - // Results of workers not consistent - WORKER_RESULTS_INCONSISTENT_ERROR = 41; - - // Unknown error. - UNKNOWN_ERROR = 101; - - // Fatal error. - // Means the service is not recoverable. - // e.g. engine socket closed in grpc connection. Usually means engine has crashed. - FATAL_ERROR = 102; -} diff --git a/proto/message.proto b/proto/message.proto index 2a7bbf6ab54b..bf14f92902dd 100644 --- a/proto/message.proto +++ b/proto/message.proto @@ -17,7 +17,7 @@ syntax = "proto3"; package gs.rpc; -import "error_codes.proto"; +import "error/coordinator.proto"; import "op_def.proto"; import "types.proto"; diff --git a/proto/op_def.proto b/proto/op_def.proto index 1cd9c6b98d8b..1fc825461507 100644 --- a/proto/op_def.proto +++ b/proto/op_def.proto @@ -18,7 +18,7 @@ syntax = "proto3"; package gs.rpc; import "attr_value.proto"; -import "error_codes.proto"; +import "error/coordinator.proto"; import "graph_def.proto"; import "types.proto"; diff --git a/proto/proto_generator.py b/proto/proto_generator.py index 85344d85da9a..ff4227f4b02d 100644 --- a/proto/proto_generator.py +++ b/proto/proto_generator.py @@ -38,6 +38,8 @@ def create_path(path): def cpp_out(relative_dir, output_dir): files = gather_all_proto(relative_dir) + error_files = gather_all_proto(os.path.join(relative_dir, "error"), suffix="coordinator.proto") + files.extend(error_files) for proto_file in files: cmd = [ shutil.which("protoc"), @@ -54,7 +56,9 @@ def cpp_out(relative_dir, output_dir): def python_out(relative_dir, output_dir): files = gather_all_proto(relative_dir) groot_files = gather_all_proto(os.path.join(relative_dir, "groot", "sdk")) + error_files = gather_all_proto(os.path.join(relative_dir, "error")) files.extend(groot_files) + files.extend(error_files) for proto_file in files: cmd = [ sys.executable, diff --git a/python/graphscope/client/rpc.py b/python/graphscope/client/rpc.py index cf65ca5932d2..58f5c131a5fa 100644 --- a/python/graphscope/client/rpc.py +++ b/python/graphscope/client/rpc.py @@ -30,8 +30,8 @@ from graphscope.client.utils import handle_grpc_error from graphscope.client.utils import suppress_grpc_error from graphscope.proto import coordinator_service_pb2_grpc -from graphscope.proto import error_codes_pb2 from graphscope.proto import message_pb2 +from graphscope.proto.error import coordinator_pb2 from graphscope.version import __version__ logger = logging.getLogger("graphscope") @@ -184,10 +184,11 @@ def _run_step_impl(self, dag_def): response = self._grpc_utils.parse_runstep_responses( self._stub.RunStep(runstep_requests) ) - if response.code != error_codes_pb2.OK: + + if response.code != coordinator_pb2.OK: logger.error( "Runstep failed with code: %s, message: %s", - error_codes_pb2.Code.Name(response.code), + coordinator_pb2.Code.Name(response.code), response.error_msg, ) if response.full_exception: diff --git a/python/graphscope/framework/errors.py b/python/graphscope/framework/errors.py index 164b22721d7c..d627781597c3 100644 --- a/python/graphscope/framework/errors.py +++ b/python/graphscope/framework/errors.py @@ -18,8 +18,8 @@ import inspect -from graphscope.proto import error_codes_pb2 from graphscope.proto import op_def_pb2 +from graphscope.proto.error import coordinator_pb2 # All kinds of GraphScope error. __all__ = [ @@ -49,7 +49,7 @@ class _ReprableString(str): """A special class that prevents `repr()` adding extra `""` to `str`. - It is used to optimize the user experiences to preseve `\n` when printing exceptions. + It is used to optimize the user experiences to preserve `\n` when printing exceptions. """ def __repr__(self) -> str: @@ -149,27 +149,69 @@ class RetriesExceededError(GSError): pass +class CancelledError(GSError): + pass + + +class ResourceExhaustedError(GSError): + pass + + +class PortInUseError(GSError): + pass + + +class UnsupportedOperationError(GSError): + pass + + +class IllegalStateError(GSError): + pass + + +class NetworkFailureError(GSError): + pass + + +class ConnectionError(GSError): + pass + + +class GremlinQueryError(GSError): + pass + + +class WorkerResultsInconsistentError(GSError): + pass + + _gs_error_types = { - error_codes_pb2.TIMEOUT_ERROR: TimeoutError, - error_codes_pb2.NOT_FOUND_ERROR: NotFoundError, - error_codes_pb2.CONNECTION_ERROR: ConnectionError, - error_codes_pb2.VINEYARD_ERROR: VineyardError, - error_codes_pb2.COMPILATION_ERROR: CompilationError, - error_codes_pb2.ALREADY_EXISTS_ERROR: AlreadyExistsError, - error_codes_pb2.UDF_INTERNAL_ERROR: UDFInternalError, - error_codes_pb2.UNAVAILABLE_ERROR: UnavailableError, - error_codes_pb2.INVALID_ARGUMENT_ERROR: InvalidArgumentError, - error_codes_pb2.PERMISSION_DENIED_ERROR: PermissionDeniedError, - error_codes_pb2.NETWORK_ERROR: NetworkError, - error_codes_pb2.K8S_ERROR: K8sError, - error_codes_pb2.UNIMPLEMENTED_ERROR: UnimplementedError, - error_codes_pb2.ANALYTICAL_ENGINE_INTERNAL_ERROR: AnalyticalEngineInternalError, - error_codes_pb2.INTERACTIVE_ENGINE_INTERNAL_ERROR: InteractiveEngineInternalError, - error_codes_pb2.LEARNING_ENGINE_INTERNAL_ERROR: LearningEngineInternalError, - error_codes_pb2.COORDINATOR_INTERNAL_ERROR: CoordinatorInternalError, - error_codes_pb2.UNKNOWN_ERROR: UnknownError, - error_codes_pb2.FATAL_ERROR: FatalError, - error_codes_pb2.RETRIES_EXCEEDED_ERROR: RetriesExceededError, + coordinator_pb2.CANCELLED: CancelledError, + coordinator_pb2.INVALID_ARGUMENT: InvalidArgumentError, + coordinator_pb2.TIMEOUT: TimeoutError, + coordinator_pb2.NOT_FOUND: NotFoundError, + coordinator_pb2.ALREADY_EXISTS: AlreadyExistsError, + coordinator_pb2.RESOURCE_EXHAUSTED: ResourceExhaustedError, + coordinator_pb2.UNIMPLEMENTED: UnimplementedError, + coordinator_pb2.PERMISSION_DENIED: PermissionDeniedError, + coordinator_pb2.COMPILATION_FAILURE: CompilationError, + coordinator_pb2.PORT_IN_USE: PortInUseError, + coordinator_pb2.UNSUPPORTED_OPERATION: UnsupportedOperationError, + coordinator_pb2.ILLEGAL_STATE: IllegalStateError, + coordinator_pb2.NETWORK_FAILURE: NetworkFailureError, + coordinator_pb2.CONNECTION_ERROR: ConnectionError, + coordinator_pb2.VINEYARD_ERROR: VineyardError, + coordinator_pb2.UDF_INTERNAL_ERROR: UDFInternalError, + coordinator_pb2.K8S_ERROR: K8sError, + coordinator_pb2.GREMLIN_QUERY_ERROR: GremlinQueryError, + coordinator_pb2.RETRIES_EXCEEDED_ERROR: RetriesExceededError, + coordinator_pb2.ANALYTICAL_ENGINE_INTERNAL_ERROR: AnalyticalEngineInternalError, + coordinator_pb2.INTERACTIVE_ENGINE_INTERNAL_ERROR: InteractiveEngineInternalError, + coordinator_pb2.LEARNING_ENGINE_INTERNAL_ERROR: LearningEngineInternalError, + coordinator_pb2.COORDINATOR_INTERNAL_ERROR: CoordinatorInternalError, + coordinator_pb2.WORKER_RESULTS_INCONSISTENT_ERROR: WorkerResultsInconsistentError, + coordinator_pb2.UNKNOWN_ERROR: UnknownError, + coordinator_pb2.FATAL_ERROR: FatalError, } diff --git a/python/graphscope/proto/error/__init__.py b/python/graphscope/proto/error/__init__.py new file mode 100644 index 000000000000..618dda69a7be --- /dev/null +++ b/python/graphscope/proto/error/__init__.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# Copyright 2020 Alibaba Group Holding Limited. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#