hotfix remove notna from ratings first

threnjen · Jan 11, 2025 · 2026a81 · 2026a81
1 parent ef9c8ef
commit 2026a81
Showing 1 changed file with 1 addition and 0 deletions.
diff --git a/modules/ratings_data_cleaner/main.py b/modules/ratings_data_cleaner/main.py
@@ -118,6 +118,7 @@ def _create_table_from_data(self, all_entries: dict[list]) -> pd.DataFrame:
 
     def _create_quality_review_table(self, df: pd.DataFrame) -> pd.DataFrame:
         """Create a cleaned and refined table of data"""
+        df = df[df["value"].notna()]
 
         df["value"] = df["value"].replace(r"[^A-Za-z0-9 ]+", "", regex=True)
         df["value"] = df["value"].str.lower().apply(lambda x: filter_stopwords(x))