From d8f9cec9de97795109fb99e83ddc0c0d84972cd6 Mon Sep 17 00:00:00 2001 From: Uche Madu Date: Wed, 18 Oct 2023 21:27:54 +0100 Subject: [PATCH] fix pyspark logic --- pyspark-scripts/config.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pyspark-scripts/config.py b/pyspark-scripts/config.py index 52481a1..f42ef6d 100644 --- a/pyspark-scripts/config.py +++ b/pyspark-scripts/config.py @@ -1,6 +1,9 @@ +from dags.user_analytics import BUCKET_NAME + + GCS_BUCKET = "deb-capstone" MOVIE_FILES = "{}/project-data/movie_reviews".format(GCS_BUCKET) -MOVIES_METADATA_FILE_PATH = "{}/processed_files_metadata.txt".format(MOVIE_FILES) +MOVIES_METADATA_FILE_PATH = "{}/project-data/movie_reviews/processed_files_metadata.txt".format(GCS_BUCKET) MODEL_DIR = "{}/models/sentiment_spark_nlp".format(GCS_BUCKET) # My HuggingFace sentiment model fine-tuned using IMDb movie reviews dataset. @@ -15,4 +18,4 @@ BQ_LOG_REVIEWS_TABLE = "review_logs" LOG_FILES = "{}/project-data/log_reviews".format(GCS_BUCKET) -LOG_METADATA_FILE_PATH = "{}/processed_files_metadata.txt".format(LOG_FILES) +LOG_METADATA_FILE_PATH = "{}/project-data/log_reviews/processed_files_metadata.txt".format(GCS_BUCKET)