hotfix ratings cleaner

threnjen · Jan 11, 2025 · 14f52cc · 14f52cc
1 parent 2026a81
commit 14f52cc
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/modules/ratings_data_cleaner/main.py b/modules/ratings_data_cleaner/main.py
@@ -122,6 +122,7 @@ def _create_quality_review_table(self, df: pd.DataFrame) -> pd.DataFrame:
 
         df["value"] = df["value"].replace(r"[^A-Za-z0-9 ]+", "", regex=True)
         df["value"] = df["value"].str.lower().apply(lambda x: filter_stopwords(x))
+        df["value"] = df["value"].str.replace("  ", " ")
 
         df["quality_review"] = df["value"].apply(evaluate_quality_words_over_thresh)
         df = df[df["quality_review"] == True]