From 22f312489a98cde58a36f1fec36f02a98f832638 Mon Sep 17 00:00:00 2001 From: Ghulam Shabbir Khan Date: Thu, 20 Jun 2024 12:31:56 +0500 Subject: [PATCH 1/3] Fixed issues in load_as_spark Fixed issue for when initiating load_as_spark() in case when a spark session is already initiated in separate thread and cannot be started in current thread. This usually arises when using the method in django rest api. --- python/delta_sharing/delta_sharing.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/python/delta_sharing/delta_sharing.py b/python/delta_sharing/delta_sharing.py index df727074f..af24aa7d2 100644 --- a/python/delta_sharing/delta_sharing.py +++ b/python/delta_sharing/delta_sharing.py @@ -150,11 +150,7 @@ def load_as_spark( except ImportError: raise ImportError("Unable to import pyspark. `load_as_spark` requires PySpark.") - spark = SparkSession.getActiveSession() - assert spark is not None, ( - "No active SparkSession was found. " - "`load_as_spark` requires running in a PySpark application." - ) + spark = SparkSession.active() df = spark.read.format("deltaSharing") if version is not None: df.option("versionAsOf", version) From a504febee25f22da9b42059dc41e5883007e85e4 Mon Sep 17 00:00:00 2001 From: Ghulam Shabbir Khan Date: Thu, 20 Jun 2024 12:35:09 +0500 Subject: [PATCH 2/3] Revert "Fixed issues in load_as_spark" This reverts commit 22f312489a98cde58a36f1fec36f02a98f832638. --- python/delta_sharing/delta_sharing.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/python/delta_sharing/delta_sharing.py b/python/delta_sharing/delta_sharing.py index af24aa7d2..df727074f 100644 --- a/python/delta_sharing/delta_sharing.py +++ b/python/delta_sharing/delta_sharing.py @@ -150,7 +150,11 @@ def load_as_spark( except ImportError: raise ImportError("Unable to import pyspark. `load_as_spark` requires PySpark.") - spark = SparkSession.active() + spark = SparkSession.getActiveSession() + assert spark is not None, ( + "No active SparkSession was found. " + "`load_as_spark` requires running in a PySpark application." + ) df = spark.read.format("deltaSharing") if version is not None: df.option("versionAsOf", version) From 4f5324edbbcf48dc2bd9318dad4004942a2157e2 Mon Sep 17 00:00:00 2001 From: Ghulam Shabbir Khan Date: Thu, 20 Jun 2024 12:38:47 +0500 Subject: [PATCH 3/3] Fixed issues in load_as_spark Fixed issue for when initiating load_as_spark() in case when a spark session is already initiated in separate thread and cannot be started in current thread. This usually arises when using the method in Django rest api. --- python/delta_sharing/delta_sharing.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/python/delta_sharing/delta_sharing.py b/python/delta_sharing/delta_sharing.py index df727074f..af24aa7d2 100644 --- a/python/delta_sharing/delta_sharing.py +++ b/python/delta_sharing/delta_sharing.py @@ -150,11 +150,7 @@ def load_as_spark( except ImportError: raise ImportError("Unable to import pyspark. `load_as_spark` requires PySpark.") - spark = SparkSession.getActiveSession() - assert spark is not None, ( - "No active SparkSession was found. " - "`load_as_spark` requires running in a PySpark application." - ) + spark = SparkSession.active() df = spark.read.format("deltaSharing") if version is not None: df.option("versionAsOf", version)