update pyspark to 3.5.0

apache · Jan 25, 2024 · 58287d6 · 58287d6
1 parent 0f08806
commit 58287d6
Show file tree

Hide file tree

Showing 5 changed files with 17 additions and 43 deletions.
diff --git a/poetry.lock b/poetry.lock
diff --git a/pyproject.toml b/pyproject.toml
@@ -82,7 +82,7 @@ requests-mock = "1.11.0"
 moto = { version = "^4.2.13", extras = ["server"] }
 typing-extensions = "4.9.0"
 pytest-mock = "3.12.0"
-pyspark = "3.4.2"
+pyspark = "3.5.0"
 cython = "3.0.8"
 
 [[tool.mypy.overrides]]

diff --git a/tests/__init__.py b/tests/__init__.py
diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py
diff --git a/tests/integration/test_writes.py b/tests/integration/test_writes.py
@@ -230,10 +230,16 @@ def table_v1_v2_appended_with_null(session_catalog: Catalog, arrow_table_with_nu
 
 @pytest.fixture(scope="session")
 def spark() -> SparkSession:
+    import importlib.metadata
     import os
 
+    spark_version = ".".join(importlib.metadata.version("pyspark").split(".")[:2])
+    scala_version = "2.12"
+    iceberg_version = "1.4.3"
+
     os.environ["PYSPARK_SUBMIT_ARGS"] = (
-        "--packages org.apache.iceberg:iceberg-spark-runtime-3.4_2.12:1.4.0,org.apache.iceberg:iceberg-aws-bundle:1.4.0 pyspark-shell"
+        f"--packages org.apache.iceberg:iceberg-spark-runtime-{spark_version}_{scala_version}:{iceberg_version},"
+        f"org.apache.iceberg:iceberg-aws-bundle:{iceberg_version} pyspark-shell"
     )
     os.environ["AWS_REGION"] = "us-east-1"
     os.environ["AWS_ACCESS_KEY_ID"] = "admin"