[Spark] Support Gluten Vectorized Engine (lakesoul-io#374)

* initial support for gluten plugin Signed-off-by: chenxu <[email protected]> * fix memory leak caused by inconsistent arrow version Signed-off-by: chenxu <[email protected]> * remove compile time dependencies for gluten Signed-off-by: chenxu <[email protected]> * refine gluten test Signed-off-by: chenxu <[email protected]> * fix path cleanup in test Signed-off-by: chenxu <[email protected]> * fix test case for gluten Signed-off-by: chenxu <[email protected]> * temporarily disable gluten test Signed-off-by: chenxu <[email protected]> --------- Signed-off-by: chenxu <[email protected]> Co-authored-by: chenxu <[email protected]>
xuchen-plus · Dec 16, 2023 · 74c28a2 · 74c28a2
1 parent be87470
commit 74c28a2
Show file tree

Hide file tree

Showing 26 changed files with 432 additions and 139 deletions.
diff --git a/.github/workflows/deployment.yml b/.github/workflows/deployment.yml
@@ -14,9 +14,9 @@ jobs:
   build-linux-x86_64:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '8'
           distribution: 'temurin'
@@ -34,21 +34,21 @@ jobs:
           use-cross: true
           command: build
           args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features'
-      - uses: actions/upload-artifact@master
+      - uses: actions/upload-artifact@v4
         with:
           name: lakesoul-nativeio-x86_64-unknown-linux-gnu
           path: ./rust/target/x86_64-unknown-linux-gnu/release/liblakesoul_io_c.so
-      - uses: actions/upload-artifact@master
+      - uses: actions/upload-artifact@v4
         with:
           name: lakesoul-nativemetadata-x86_64-unknown-linux-gnu
           path: ./rust/target/x86_64-unknown-linux-gnu/release/liblakesoul_metadata_c.so
 
   build-windows-x86_64:
     runs-on: windows-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '8'
           distribution: 'temurin'
@@ -70,11 +70,11 @@ jobs:
         with:
           command: build
           args: '--manifest-path rust/Cargo.toml --release --all-features'
-      - uses: actions/upload-artifact@master
+      - uses: actions/upload-artifact@v4
         with:
           name: lakesoul-nativeio-x86_64-pc-windows-msvc
           path: ./rust/target/release/lakesoul_io_c.dll
-      - uses: actions/upload-artifact@master
+      - uses: actions/upload-artifact@v4
         with:
           name: lakesoul-nativemetadata-x86_64-pc-windows-msvc
           path: ./rust/target/release/lakesoul_metadata_c.dll
@@ -84,9 +84,9 @@ jobs:
     steps:
       - name: Install automake
         run: brew install automake
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '8'
           distribution: 'temurin'
@@ -108,11 +108,11 @@ jobs:
         with:
           command: build
           args: '--manifest-path rust/Cargo.toml --release --all-features'
-      - uses: actions/upload-artifact@master
+      - uses: actions/upload-artifact@v4
         with:
           name: lakesoul-nativeio-x86_64-apple-darwin
           path: ./rust/target/release/liblakesoul_io_c.dylib
-      - uses: actions/upload-artifact@master
+      - uses: actions/upload-artifact@v4
         with:
           name: lakesoul-nativemetadata-x86_64-apple-darwin
           path: ./rust/target/release/liblakesoul_metadata_c.dylib
@@ -122,33 +122,33 @@ jobs:
     runs-on: ubuntu-latest
     needs: [ build-linux-x86_64, build-windows-x86_64, build-macos-x86_64 ]
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/download-artifact@v3
+      - uses: actions/checkout@v4
+      - uses: actions/download-artifact@v4
         with:
           name: lakesoul-nativeio-x86_64-unknown-linux-gnu
           path: ./rust/target/release/
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: lakesoul-nativeio-x86_64-apple-darwin
           path: ./rust/target/release/
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: lakesoul-nativeio-x86_64-pc-windows-msvc
           path: ./rust/target/release/
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: lakesoul-nativemetadata-x86_64-unknown-linux-gnu
           path: ./rust/target/release/
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: lakesoul-nativemetadata-x86_64-apple-darwin
           path: ./rust/target/release/
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: lakesoul-nativemetadata-x86_64-pc-windows-msvc
           path: ./rust/target/release/
       - name: Set up JDK 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '8'
           distribution: 'temurin'

diff --git a/.github/workflows/flink-cdc-hdfs-test.yml b/.github/workflows/flink-cdc-hdfs-test.yml
@@ -29,9 +29,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '8'
           distribution: 'temurin'
@@ -185,7 +185,7 @@ jobs:
       - name: Upload Log
         if: always()
         continue-on-error: true
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: flink-cluster-log
           path: flink-session-cluster.log

diff --git a/.github/workflows/flink-cdc-test.yml b/.github/workflows/flink-cdc-test.yml
@@ -29,9 +29,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '8'
           distribution: 'temurin'
@@ -177,7 +177,7 @@ jobs:
       - name: Upload Log
         if: always()
         continue-on-error: true
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: flink-cluster-log
           path: flink-session-cluster.log

diff --git a/.github/workflows/maven-test.yml b/.github/workflows/maven-test.yml
@@ -29,9 +29,9 @@ jobs:
   build-rust-linux-x86_64:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '8'
           distribution: 'temurin'
@@ -49,11 +49,11 @@ jobs:
           use-cross: true
           command: build
           args: '--manifest-path rust/Cargo.toml --target x86_64-unknown-linux-gnu --release --all-features'
-      - uses: actions/upload-artifact@master
+      - uses: actions/upload-artifact@v4
         with:
           name: lakesoul-nativeio-x86_64-unknown-linux-gnu-maven-test
           path: ./rust/target/x86_64-unknown-linux-gnu/release/liblakesoul_io_c.so
-      - uses: actions/upload-artifact@master
+      - uses: actions/upload-artifact@v4
         with:
           name: lakesoul-nativemetadata-x86_64-unknown-linux-gnu-maven-test
           path: ./rust/target/x86_64-unknown-linux-gnu/release/liblakesoul_metadata_c.so
@@ -84,9 +84,9 @@ jobs:
           - 5432:5432
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '8'
           distribution: 'temurin'
@@ -101,11 +101,11 @@ jobs:
         with:
           version: "23.x"
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: lakesoul-nativemetadata-x86_64-unknown-linux-gnu-maven-test
           path: ./rust/target/release/
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: lakesoul-nativeio-x86_64-unknown-linux-gnu-maven-test
           path: ./rust/target/release/
@@ -119,7 +119,7 @@ jobs:
       - name: Upload Test Report
         if: always()
         continue-on-error: true
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: maven-test-report-artifact-spark-1
           path: lakesoul-spark/target/site
@@ -152,9 +152,9 @@ jobs:
           - 5432:5432
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '8'
           distribution: 'temurin'
@@ -169,11 +169,11 @@ jobs:
         with:
           version: "23.x"
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: lakesoul-nativemetadata-x86_64-unknown-linux-gnu-maven-test
           path: ./rust/target/release/
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: lakesoul-nativeio-x86_64-unknown-linux-gnu-maven-test
           path: ./rust/target/release/
@@ -187,7 +187,7 @@ jobs:
       - name: Upload Test Report
         if: always()
         continue-on-error: true
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: maven-test-report-artifact-spark-2
           path: lakesoul-spark/target/site
@@ -220,9 +220,9 @@ jobs:
           - 5432:5432
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '8'
           distribution: 'temurin'
@@ -249,11 +249,11 @@ jobs:
           $HADOOP_HOME/sbin/stop-dfs.sh
           $HADOOP_HOME/sbin/start-dfs.sh
           $HADOOP_HOME/bin/hadoop fs -chmod -R 777 /
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: lakesoul-nativemetadata-x86_64-unknown-linux-gnu-maven-test
           path: ./rust/target/release/
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: lakesoul-nativeio-x86_64-unknown-linux-gnu-maven-test
           path: ./rust/target/release/
@@ -267,7 +267,7 @@ jobs:
       - name: Upload Test Report
         if: always()
         continue-on-error: true
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: maven-test-report-artifact-spark-2
           path: lakesoul-spark/target/site
@@ -300,9 +300,9 @@ jobs:
           - 5432:5432
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '8'
           distribution: 'temurin'
@@ -317,11 +317,11 @@ jobs:
         with:
           version: "23.x"
           repo-token: ${{ secrets.GITHUB_TOKEN }}
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: lakesoul-nativemetadata-x86_64-unknown-linux-gnu-maven-test
           path: ./rust/target/release/
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: lakesoul-nativeio-x86_64-unknown-linux-gnu-maven-test
           path: ./rust/target/release/
@@ -335,7 +335,7 @@ jobs:
       - name: Upload Test Report
         if: always()
         continue-on-error: true
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: maven-test-report-artifact-flink-1
           path: lakesoul-flink/target/site
@@ -368,9 +368,9 @@ jobs:
           - 5432:5432
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
       - name: Set up JDK 8
-        uses: actions/setup-java@v3
+        uses: actions/setup-java@v4
         with:
           java-version: '8'
           distribution: 'temurin'
@@ -405,11 +405,11 @@ jobs:
           $HADOOP_HOME/sbin/stop-dfs.sh
           $HADOOP_HOME/sbin/start-dfs.sh
           $HADOOP_HOME/bin/hadoop fs -chmod -R 777 /
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: lakesoul-nativemetadata-x86_64-unknown-linux-gnu-maven-test
           path: ./rust/target/release/
-      - uses: actions/download-artifact@v3
+      - uses: actions/download-artifact@v4
         with:
           name: lakesoul-nativeio-x86_64-unknown-linux-gnu-maven-test
           path: ./rust/target/release/
@@ -423,7 +423,7 @@ jobs:
       - name: Upload Test Report
         if: always()
         continue-on-error: true
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: maven-test-report-artifact-flink-1
           path: lakesoul-flink/target/site