[feature](merge-cloud) Publish meta service and recycler service

Co-authored-by: plat1ko <[email protected]> Co-authored-by: Gavin Chou <[email protected]> Co-authored-by: Xiaocc <[email protected]> Co-authored-by: deardeng <[email protected]> Co-authored-by: meiyi <[email protected]> Co-authored-by: Luwei <[email protected]> Co-authored-by: Xin Liao <[email protected]> Co-authored-by: Lei Zhang <[email protected]> Co-authored-by: Lightman <[email protected]> Co-authored-by: YueW <[email protected]> Co-authored-by: HHoflittlefish777 <[email protected]> Co-authored-by: Siyang Tang <[email protected]> Co-authored-by: bobhan1 <[email protected]> Co-authored-by: Kidd <[email protected]> Co-authored-by: Tanya-W <[email protected]> Co-authored-by: panDing19 <[email protected]> Co-authored-by: w41ter <[email protected]> Co-authored-by: AlexYue <[email protected]> Co-authored-by: abmdocrt <[email protected]>
w41ter · Jan 17, 2024 · f617615 · f617615
1 parent e556cff
commit f617615
Show file tree

Hide file tree

Showing 128 changed files with 52,426 additions and 4 deletions.
diff --git a/.github/workflows/cloud-clang-format.yaml b/.github/workflows/cloud-clang-format.yaml
@@ -0,0 +1,79 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+---
+name: Cloud Code Formatter
+
+on: [push, pull_request_target]
+
+jobs:
+  clang-format:
+    name: "Clang Formatter"
+    runs-on: ubuntu-latest
+    steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        if: ${{ github.event_name != 'pull_request_target' }}
+        uses: actions/checkout@v3
+        with:
+          persist-credentials: false
+
+      - name: Checkout ${{ github.ref }} ( ${{ github.event.pull_request.head.sha }} )
+        if: ${{ github.event_name == 'pull_request_target' }}
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Checkout paths-filter
+        run: |
+          rm -rf ./.github/actions/paths-filter
+          git clone https://github.com/dorny/paths-filter .github/actions/paths-filter
+
+          pushd .github/actions/paths-filter &>/dev/null
+          git checkout 4512585405083f25c027a35db413c2b3b9006d50
+          popd &>/dev/null
+
+      - name: Paths filter
+        uses: ./.github/actions/paths-filter
+        id: filter
+        with:
+          filters: |
+            cloud_changes:
+              - 'cloud/src/**'
+              - 'cloud/test/**'
+
+      - name: Checkout clang-format-lint-action
+        run: |
+          rm -rf ./.github/actions/clang-format-lint-action
+          git clone https://github.com/DoozyX/clang-format-lint-action .github/actions/clang-format-lint-action
+
+          pushd .github/actions/clang-format-lint-action &>/dev/null
+          git checkout 6adbe14579e5b8e19eb3e31e5ff2479f3bd302c7
+          popd &>/dev/null
+
+      - name: "Format it!"
+        if: ${{ steps.filter.outputs.cloud_changes == 'true' }}
+        uses: ./.github/actions/clang-format-lint-action
+        id: cloud_clang_format
+        with:
+          source: "cloud/src cloud/test"
+          clangFormatVersion: 16
+          inplace: False
+
+      - name: Ignore it!
+        if: ${{ steps.filter.outputs.cloud_changes == 'false' }}
+        run: echo 'No need to check.'
diff --git a/.github/workflows/code-checks.yml b/.github/workflows/code-checks.yml
@@ -104,7 +104,7 @@ jobs:
             popd
 
             export PATH="${DEFAULT_DIR}/ldb-toolchain/bin/:$(pwd)/thirdparty/installed/bin/:${PATH}"
-            DISABLE_JAVA_UDF=ON DORIS_TOOLCHAIN=clang ENABLE_PCH=OFF OUTPUT_BE_BINARY=0 ./build.sh --be
+            DISABLE_JAVA_UDF=ON DORIS_TOOLCHAIN=clang ENABLE_PCH=OFF OUTPUT_BE_BINARY=0 ./build.sh --be --cloud
           fi
 
           echo "should_check=${{ steps.filter.outputs.be_changes }}" >>${GITHUB_OUTPUT}

diff --git a/.gitignore b/.gitignore
@@ -99,6 +99,11 @@ be/.devcontainer/
 be/src/apache-orc/
 zoneinfo/
 
+# Cloud
+cloud/build*/
+cloud/cmake-build*/
+cloud/ut_build*/
+
 ## tools
 tools/ssb-tools/ssb-data/
 tools/ssb-tools/ssb-dbgen/

diff --git a/build.sh b/build.sh
@@ -42,6 +42,7 @@ Usage: $0 <options>
      --fe                   build Frontend and Spark DPP application. Default ON.
      --be                   build Backend. Default ON.
      --meta-tool            build Backend meta tool. Default OFF.
+     --cloud                build Cloud. Default OFF.
      --index-tool           build Backend inverted index tool. Default OFF.
      --broker               build Broker. Default ON.
      --audit                build audit loader. Default ON.
@@ -61,6 +62,7 @@ Usage: $0 <options>
     $0                                      build all
     $0 --be                                 build Backend
     $0 --meta-tool                          build Backend meta tool
+    $0 --cloud                              build Cloud
     $0 --index-tool                         build Backend inverted index tool
     $0 --fe --clean                         clean and build Frontend and Spark Dpp application
     $0 --fe --be --clean                    clean and build Frontend, Spark Dpp application and Backend
@@ -117,6 +119,7 @@ if ! OPTS="$(getopt \
     -o '' \
     -l 'fe' \
     -l 'be' \
+    -l 'cloud' \
     -l 'broker' \
     -l 'audit' \
     -l 'meta-tool' \
@@ -138,6 +141,7 @@ eval set -- "${OPTS}"
 PARALLEL="$(($(nproc) / 4 + 1))"
 BUILD_FE=0
 BUILD_BE=0
+BUILD_CLOUD=0
 BUILD_BROKER=0
 BUILD_AUDIT=0
 BUILD_META_TOOL='OFF'
@@ -154,6 +158,7 @@ if [[ "$#" == 1 ]]; then
     # default
     BUILD_FE=1
     BUILD_BE=1
+
     BUILD_BROKER=1
     BUILD_AUDIT=1
     BUILD_META_TOOL='OFF'
@@ -177,6 +182,10 @@ else
             BUILD_BE_JAVA_EXTENSIONS=1
             shift
             ;;
+        --cloud)
+            BUILD_CLOUD=1
+            shift
+            ;;
         --broker)
             BUILD_BROKER=1
             shift
@@ -244,6 +253,7 @@ else
     if [[ "${PARAMETER_COUNT}" -eq 3 ]] && [[ "${PARAMETER_FLAG}" -eq 1 ]]; then
         BUILD_FE=1
         BUILD_BE=1
+        BUILD_CLOUD=1
         BUILD_BROKER=1
         BUILD_AUDIT=1
         BUILD_META_TOOL='ON'
@@ -307,7 +317,7 @@ update_submodule() {
     fi
 }
 
-if [[ "${CLEAN}" -eq 1 && "${BUILD_BE}" -eq 0 && "${BUILD_FE}" -eq 0 && "${BUILD_SPARK_DPP}" -eq 0 ]]; then
+if [[ "${CLEAN}" -eq 1 && "${BUILD_BE}" -eq 0 && "${BUILD_FE}" -eq 0 && "${BUILD_SPARK_DPP}" -eq 0 && ${BUILD_CLOUD} -eq 0 ]]; then
     clean_gensrc
     clean_be
     clean_fe
@@ -415,6 +425,7 @@ fi
 echo "Get params:
     BUILD_FE                    -- ${BUILD_FE}
     BUILD_BE                    -- ${BUILD_BE}
+    BUILD_CLOUD                 -- ${BUILD_CLOUD}
     BUILD_BROKER                -- ${BUILD_BROKER}
     BUILD_AUDIT                 -- ${BUILD_AUDIT}
     BUILD_META_TOOL             -- ${BUILD_META_TOOL}
@@ -541,6 +552,42 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then
     cd "${DORIS_HOME}"
 fi
 
+# Clean and build cloud
+if [[ "${BUILD_CLOUD}" -eq 1 ]]; then
+    if [[ -e "${DORIS_HOME}/gensrc/build/gen_cpp/cloud_version.h" ]]; then
+        rm -f "${DORIS_HOME}/gensrc/build/gen_cpp/cloud_version.h"
+    fi
+    CMAKE_BUILD_TYPE="${BUILD_TYPE:-Release}"
+    echo "Build Cloud: ${CMAKE_BUILD_TYPE}"
+    CMAKE_BUILD_DIR="${DORIS_HOME}/cloud/build_${CMAKE_BUILD_TYPE}"
+    if [[ "${CLEAN}" -eq 1 ]]; then
+        rm -rf "${CMAKE_BUILD_DIR}"
+        echo "clean cloud"
+    fi
+    MAKE_PROGRAM="$(command -v "${BUILD_SYSTEM}")"
+    echo "-- Make program: ${MAKE_PROGRAM}"
+    echo "-- Extra cxx flags: ${EXTRA_CXX_FLAGS:-}"
+    mkdir -p "${CMAKE_BUILD_DIR}"
+    cd "${CMAKE_BUILD_DIR}"
+    "${CMAKE_CMD}" -G "${GENERATOR}" \
+        -DCMAKE_MAKE_PROGRAM="${MAKE_PROGRAM}" \
+        -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \
+        -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" \
+        -DMAKE_TEST=OFF \
+        "${CMAKE_USE_CCACHE}" \
+        -DUSE_LIBCPP="${USE_LIBCPP}" \
+        -DSTRIP_DEBUG_INFO="${STRIP_DEBUG_INFO}" \
+        -DUSE_DWARF="${USE_DWARF}" \
+        -DUSE_JEMALLOC="${USE_JEMALLOC}" \
+        -DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \
+        -DBUILD_CHECK_META="${BUILD_CHECK_META:-OFF}" \
+        "${DORIS_HOME}/cloud/"
+    "${BUILD_SYSTEM}" -j "${PARALLEL}"
+    "${BUILD_SYSTEM}" install
+    cd "${DORIS_HOME}"
+    echo "Build cloud done"
+fi
+
 if [[ "${BUILD_DOCS}" = "ON" ]]; then
     # Build docs, should be built before Frontend
     echo "Build docs"
@@ -754,6 +801,11 @@ if [[ "${BUILD_AUDIT}" -eq 1 ]]; then
     cd "${DORIS_HOME}"
 fi
 
+if [[ ${BUILD_CLOUD} -eq 1 ]]; then
+    rm -rf "${DORIS_HOME}/output/ms"
+    cp -r -p "${DORIS_HOME}/cloud/output" "${DORIS_HOME}/output/ms"
+fi
+
 echo "***************************************"
 echo "Successfully build Doris"
 echo "***************************************"