Add CI for spark 3.4 (#47)

Signed-off-by: Weichen Xu <[email protected]>

.github/workflows/main.yml

@@ -8,7 +8,7 @@ jobs:
       matrix:
         PYTHON_VERSION: ["3.7", "3.8", "3.9", "3.10"]
         PIN_MODE: [false, true]
-        PYSPARK_VERSION: ["3.0.3", "3.1.3", "3.2.3", "3.3.1"]
+        PYSPARK_VERSION: ["3.0.3", "3.1.3", "3.2.3", "3.3.2", "3.4.0"]
         exclude:
           - PYSPARK_VERSION: "3.0.3"
             PIN_MODE: true
@@ -25,7 +25,11 @@ jobs:
       - name: Install python packages
         run: |
           pip install joblib>=0.14.0 scikit-learn>=0.23.1 pytest pylint
-          pip install pyspark==${{ matrix.PYSPARK_VERSION }}
+          if [ "${{ matrix.PYSPARK_VERSION }}" = "3.4.0" ]; then
+              pip install https://dist.apache.org/repos/dist/dev/spark/v3.4.0-rc1-bin/pyspark-3.4.0.tar.gz
+          else
+              pip install pyspark==${{ matrix.PYSPARK_VERSION }}
+          fi;
       - name: Run pylint
         run: |
           ./run-pylint.sh