From 85d0beb2641a66de84293f8b04f753f4acae8e9e Mon Sep 17 00:00:00 2001 From: Uche Madu Date: Thu, 19 Oct 2023 05:08:48 +0100 Subject: [PATCH] fix pyspark logic --- pyspark-scripts/process_logs.py | 2 +- pyspark-scripts/process_movies.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyspark-scripts/process_logs.py b/pyspark-scripts/process_logs.py index 0384c5b..3908872 100644 --- a/pyspark-scripts/process_logs.py +++ b/pyspark-scripts/process_logs.py @@ -56,7 +56,7 @@ def main(): # Save the DataFrame to BigQuery (log_df_parsed.write.format("bigquery") .option("temporaryGcsBucket", GCS_BUCKET) - .option("partitionField", "log_date") + # .option("partitionField", "day") .mode("append") .save(f"{BQ_DATASET_NAME}.{BQ_LOG_REVIEWS_TABLE}") ) diff --git a/pyspark-scripts/process_movies.py b/pyspark-scripts/process_movies.py index 40577ba..6d2e8d4 100644 --- a/pyspark-scripts/process_movies.py +++ b/pyspark-scripts/process_movies.py @@ -204,7 +204,7 @@ def main() -> None: # Save the DataFrame to BigQuery (result_df.write.format("bigquery") .option("temporaryGcsBucket", GCS_BUCKET) - .option("partitionField", "insert_date") + # .option("partitionField", "insert_date") .mode("append") .save(f"{BQ_DATASET_NAME}.{BQ_MOVIE_REVIEWS_TABLE}") )