fix pyspark logic

uche-madu · Oct 18, 2023 · d8f9cec · d8f9cec
1 parent 734a3cc
commit d8f9cec
Showing 1 changed file with 5 additions and 2 deletions.
diff --git a/pyspark-scripts/config.py b/pyspark-scripts/config.py
@@ -1,6 +1,9 @@
+from dags.user_analytics import BUCKET_NAME
+
+
 GCS_BUCKET = "deb-capstone"
 MOVIE_FILES = "{}/project-data/movie_reviews".format(GCS_BUCKET)
-MOVIES_METADATA_FILE_PATH = "{}/processed_files_metadata.txt".format(MOVIE_FILES)
+MOVIES_METADATA_FILE_PATH = "{}/project-data/movie_reviews/processed_files_metadata.txt".format(GCS_BUCKET)
 MODEL_DIR = "{}/models/sentiment_spark_nlp".format(GCS_BUCKET)
 
 # My HuggingFace sentiment model fine-tuned using IMDb movie reviews dataset.
@@ -15,4 +18,4 @@
 BQ_LOG_REVIEWS_TABLE = "review_logs"
 
 LOG_FILES = "{}/project-data/log_reviews".format(GCS_BUCKET)
-LOG_METADATA_FILE_PATH = "{}/processed_files_metadata.txt".format(LOG_FILES)
+LOG_METADATA_FILE_PATH = "{}/project-data/log_reviews/processed_files_metadata.txt".format(GCS_BUCKET)