fix pyspark logic

uche-madu · Oct 19, 2023 · 85d0beb · 85d0beb
1 parent 40e268a
commit 85d0beb
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 2 deletions.
diff --git a/pyspark-scripts/process_logs.py b/pyspark-scripts/process_logs.py
@@ -56,7 +56,7 @@ def main():
     # Save the DataFrame to BigQuery
     (log_df_parsed.write.format("bigquery")
         .option("temporaryGcsBucket", GCS_BUCKET)
-        .option("partitionField", "log_date")
+        # .option("partitionField", "day")
         .mode("append")
         .save(f"{BQ_DATASET_NAME}.{BQ_LOG_REVIEWS_TABLE}")
         )

diff --git a/pyspark-scripts/process_movies.py b/pyspark-scripts/process_movies.py
@@ -204,7 +204,7 @@ def main() -> None:
         # Save the DataFrame to BigQuery
         (result_df.write.format("bigquery")
             .option("temporaryGcsBucket", GCS_BUCKET)
-            .option("partitionField", "insert_date")
+            # .option("partitionField", "insert_date")
             .mode("append")
             .save(f"{BQ_DATASET_NAME}.{BQ_MOVIE_REVIEWS_TABLE}")
             )