Merge pull request #43 from uche-madu/dev

Dev
uche-madu · Oct 19, 2023 · 26f08b2 · 26f08b2
2 parents eac0d9f + 85d0beb
commit 26f08b2
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 4 deletions.
diff --git a/pyspark-scripts/process_logs.py b/pyspark-scripts/process_logs.py
@@ -55,9 +55,10 @@ def main():
 
     # Save the DataFrame to BigQuery
     (log_df_parsed.write.format("bigquery")
-        .option("table", f"{BQ_DATASET_NAME}.{BQ_LOG_REVIEWS_TABLE}")
+        .option("temporaryGcsBucket", GCS_BUCKET)
+        # .option("partitionField", "day")
         .mode("append")
-        .save()
+        .save(f"{BQ_DATASET_NAME}.{BQ_LOG_REVIEWS_TABLE}")
         )
 
 if __name__ == "__main__":

diff --git a/pyspark-scripts/process_movies.py b/pyspark-scripts/process_movies.py
@@ -203,10 +203,12 @@ def main() -> None:
 
         # Save the DataFrame to BigQuery
         (result_df.write.format("bigquery")
-            .option("table", f"{BQ_DATASET_NAME}.{BQ_MOVIE_REVIEWS_TABLE}")
+            .option("temporaryGcsBucket", GCS_BUCKET)
+            # .option("partitionField", "insert_date")
             .mode("append")
-            .save()
+            .save(f"{BQ_DATASET_NAME}.{BQ_MOVIE_REVIEWS_TABLE}")
             )
 
+
 if __name__ == "__main__":
     main()