Enable more Ruff rules (#231)

My IDE was highlighting additional warnings not covered by Ruff, so I
added some more rules. 90% of this was autofixed, the rest was
straightforward and useful. I even found an issue with a test because of
it.

.pre-commit-config.yaml

@@ -12,7 +12,11 @@ repos:
                 fondant/.*|
                 tests/.*|
             )$
-        args: [--fix, --exit-non-zero-on-fix]
+        args: [
+          "--target-version=py38",
+          "--fix",
+          "--exit-non-zero-on-fix",
+        ]
 
 
   - repo: https://github.com/PyCQA/bandit

components/caption_images/src/main.py

@@ -40,7 +40,7 @@ def caption_image_batch(
     *,
     model: BlipForConditionalGeneration,
     processor: BlipProcessor,
-    max_new_tokens: int
+    max_new_tokens: int,
 ) -> pd.Series:
     """Caption a batch of images."""
     input_batch = torch.cat(image_batch.tolist())
@@ -67,7 +67,7 @@ def transform(self, dataframe: pd.DataFrame) -> pd.DataFrame:
         images = dataframe["images"]["data"].apply(
             process_image,
             processor=self.processor,
-            device=self.device
+            device=self.device,
         )
 
         results: t.List[pd.Series] = []
@@ -78,8 +78,8 @@ def transform(self, dataframe: pd.DataFrame) -> pd.DataFrame:
                         batch,
                         model=self.model,
                         processor=self.processor,
-                        max_new_tokens=self.max_new_tokens
-                    ).T
+                        max_new_tokens=self.max_new_tokens,
+                    ).T,
                 )
 
         return pd.concat(results).to_frame(name=("captions", "text"))

components/download_images/src/main.py

@@ -49,7 +49,7 @@ def download_image(url, timeout, user_agent_token, disallowed_header_directives)
                              f"+https://github.com/rom1504/img2dataset)"
     try:
         request = urllib.request.Request(
-            url, data=None, headers={"User-Agent": user_agent_string}
+            url, data=None, headers={"User-Agent": user_agent_string},
         )
         with urllib.request.urlopen(request, timeout=timeout) as r:
             if disallowed_header_directives and is_disallowed(
@@ -77,7 +77,7 @@ def download_image_with_retry(
 ):
     for _ in range(retries + 1):
         img_stream = download_image(
-            url, timeout, user_agent_token, disallowed_header_directives
+            url, timeout, user_agent_token, disallowed_header_directives,
         )
         if img_stream is not None:
             # resize the image
@@ -114,7 +114,7 @@ def transform(self, dataframe: pd.DataFrame) -> pd.DataFrame:
         dataframe[[
             ("images", "data"),
             ("images", "width"),
-            ("images", "height")
+            ("images", "height"),
         ]] = dataframe.apply(
             lambda example: download_image_with_retry(
                 url=example["images"]["url"],

components/embedding_based_laion_retrieval/src/main.py

@@ -21,7 +21,7 @@ def setup(
             *,
             num_images: int,
             aesthetic_score: int,
-            aesthetic_weight: float
+            aesthetic_weight: float,
     ) -> None:
         """
 
@@ -54,7 +54,7 @@ async def async_query():
                 futures = [
                     loop.run_in_executor(
                         executor,
-                        functools.partial(self.client.query, embedding_input=embedding.tolist())
+                        functools.partial(self.client.query, embedding_input=embedding.tolist()),
                     )
                     for embedding in dataframe["embeddings"]["data"]
                 ]
@@ -64,7 +64,7 @@ async def async_query():
         loop.run_until_complete(async_query())
 
         results_df = pd.DataFrame(results)[["id", "url"]]
-        results_df.set_index("id", inplace=True)
+        results_df = results_df.set_index("id")
         results_df.columns = [["images"], ["url"]]
 
         return results_df

components/filter_comments/src/main.py

@@ -20,7 +20,7 @@ def transform(
         *,
         dataframe: dd.DataFrame,
         min_comments_ratio: float,
-        max_comments_ratio: float
+        max_comments_ratio: float,
     ) -> dd.DataFrame:
         """
         Args:
@@ -31,16 +31,14 @@ def transform(
             Filtered dask dataframe.
         """
         # Apply the function to the desired column and filter the DataFrame
-        filtered_df = dataframe[
+        return dataframe[
             dataframe["code_content"].map_partitions(
                 lambda example: example.map(get_comments_to_code_ratio).between(
-                    min_comments_ratio, max_comments_ratio
-                )
+                    min_comments_ratio, max_comments_ratio,
+                ),
             )
         ]
 
-        return filtered_df
-
 
 if __name__ == "__main__":
     component = FilterCommentsComponent.from_args()

components/filter_comments/src/utils/text_extraction.py

@@ -69,9 +69,7 @@ def get_comments(source: str) -> str:
     for toknum, tokval, _, _, _ in g:
         if toknum == tokenize.COMMENT:
             comments.append((toknum, tokval))
-    result = tokenize.untokenize(comments).replace("#", "")
-
-    return result
+    return tokenize.untokenize(comments).replace("#", "")
 
 
 def get_docstrings(source: str) -> t.List[str]:
@@ -88,13 +86,13 @@ def get_docstrings(source: str) -> t.List[str]:
         source = source.read()
 
     docstrings = sorted(
-        parse_docstrings(source), key=lambda x: (NODE_TYPES.get(type(x[0])), x[1])
+        parse_docstrings(source), key=lambda x: (NODE_TYPES.get(type(x[0])), x[1]),
     )
 
     grouped = groupby(docstrings, key=lambda x: NODE_TYPES.get(type(x[0])))
     results = []
     for _, group in grouped:
-        for _, name, docstring in group:
+        for _, _name, docstring in group:
             if docstring:
                 results.append(docstring)
     return results
@@ -116,7 +114,7 @@ def get_text_python(source: str, extract_comments: bool = True) -> str:
     except Exception:
         docstrings = ""
         warnings.warn(
-            "code couldn't be parsed due to compilation failure, no docstring is extracted"
+            "code couldn't be parsed due to compilation failure, no docstring is extracted",
         )
 
     if extract_comments:
@@ -142,4 +140,4 @@ def get_comments_to_code_ratio(text: str) -> float:
     """
     comments = get_text_python(text)
 
-    return len(comments) / len(text)
+    return len(comments) / len(text)

components/filter_line_length/src/main.py

@@ -20,7 +20,7 @@ def transform(
         dataframe: dd.DataFrame,
         avg_line_length_threshold: int,
         max_line_length_threshold: int,
-        alphanum_fraction_threshold: float
+        alphanum_fraction_threshold: float,
     ) -> dd.DataFrame:
         """
         Args:
@@ -31,14 +31,12 @@ def transform(
         Returns:
             Filtered dask dataframe.
         """
-        filtered_df = dataframe[
+        return dataframe[
             (dataframe["code_avg_line_length"] > avg_line_length_threshold)
             & (dataframe["code_max_line_length"] > max_line_length_threshold)
             & (dataframe["code_alphanum_fraction"] > alphanum_fraction_threshold)
         ]
 
-        return filtered_df
-
 
 if __name__ == "__main__":
     component = FilterLineLengthComponent.from_args()

components/image_cropping/src/image_crop.py

@@ -47,7 +47,7 @@ def get_image_borders(image: Image.Image) -> t.Tuple:
 
 
 def remove_borders(
-    image_bytes: bytes, cropping_threshold: int = -30, padding: int = 10
+    image_bytes: bytes, cropping_threshold: int = -30, padding: int = 10,
 ) -> bytes:
     """This method removes borders by checking the overlap between
     a color and the original image. By subtracting these two
@@ -89,12 +89,12 @@ def remove_borders(
         if image_crop.size[0] > image_crop.size[1]:
             padding = int((image_crop.size[0] - image_crop.size[1]) / 2)
             image_crop = ImageOps.expand(
-                image_crop, border=(0, padding), fill=color_common
+                image_crop, border=(0, padding), fill=color_common,
             )
         else:
             padding = int((image_crop.size[1] - image_crop.size[0]) / 2)
             image_crop = ImageOps.expand(
-                image_crop, border=(padding, 0), fill=color_common
+                image_crop, border=(padding, 0), fill=color_common,
             )
 
     # serialize image to JPEG

components/image_cropping/src/main.py

@@ -36,7 +36,7 @@ def transform(
         *,
         dataframe: dd.DataFrame,
         cropping_threshold: int = -30,
-        padding: int = 10
+        padding: int = 10,
     ) -> dd.DataFrame:
         """
         Args:

components/image_embedding/src/main.py

@@ -75,7 +75,7 @@ def transform(self, dataframe: pd.DataFrame) -> pd.DataFrame:
         images = dataframe["images"]["data"].apply(
             process_image,
             processor=self.processor,
-            device=self.device
+            device=self.device,
         )
         results: t.List[pd.Series] = []
         for batch in np.split(images, np.arange(self.batch_size, len(images), self.batch_size)):

components/image_resolution_filtering/src/main.py

@@ -12,7 +12,7 @@ class ImageFilterComponent(DaskTransformComponent):
     """Component that filters images based on height and width."""
 
     def transform(
-        self, *, dataframe: dd.DataFrame, min_width: int, min_height: int
+        self, *, dataframe: dd.DataFrame, min_width: int, min_height: int,
     ) -> dd.DataFrame:
         """
         Args:
@@ -38,4 +38,4 @@ def transform(
 
 if __name__ == "__main__":
     component = ImageFilterComponent.from_args()
-    component.run()
+    component.run()

components/load_from_hf_hub/src/main.py

@@ -35,7 +35,7 @@ def load(self,
         if image_column_names is not None:
             for image_column_name in image_column_names:
                 dask_df[image_column_name] = dask_df[image_column_name].map(
-                    lambda x: x["bytes"], meta=("bytes", bytes)
+                    lambda x: x["bytes"], meta=("bytes", bytes),
                 )
 
         # 3) Rename columns

components/pii_redaction/src/main.py

@@ -40,7 +40,7 @@ def transform(
 
         # redact PII
         # we use random replacements by default
-        with open("replacements.json", "r") as f:
+        with open("replacements.json") as f:
             replacements = json.load(f)
 
         dataframe["code_content"] = dataframe.apply(
@@ -54,7 +54,7 @@ def transform(
             meta=(None, "str"),
         )
         dataframe = dataframe.drop(
-            ["code_secrets", "code_has_secrets", "code_number_secrets"], axis=1
+            ["code_secrets", "code_has_secrets", "code_number_secrets"], axis=1,
         )
 
         return dataframe

components/pii_redaction/src/pii_detection.py

@@ -26,12 +26,12 @@ def scan_pii(text, key_detector="other"):
     if key_detector == "regex":
         # use a regex to detect keys + emails + ips
         secrets = secrets + detect_email_addresses(
-            text, tag_types={"KEY", "EMAIL", "IP_ADDRESS"}
+            text, tag_types={"KEY", "EMAIL", "IP_ADDRESS"},
         )
     else:
         # detect emails and ip addresses with regexes
         secrets = secrets + detect_email_addresses(
-            text, tag_types={"EMAIL", "IP_ADDRESS"}
+            text, tag_types={"EMAIL", "IP_ADDRESS"},
         )
         # for keys use detect-secrets tool
         secrets = secrets + detect_keys(text)

components/pii_redaction/src/pii_redaction.py

@@ -21,7 +21,6 @@
     ],
 }
 
-# providergs = ["google", "cloudfare", "alternate-dns", "quad9","open-dns", "comodo", "adguard"]
 POPULAR_DNS_SERVERS = [
     "8.8.8.8",
     "8.8.4.4",
@@ -113,7 +112,7 @@ def redact_pii_text(text, secrets, replacements, add_references=False):
         last_text = text
         for secret in secrets:
             # skip secret if it's an IP address for private networks or popular DNS servers
-            if secret["tag"] == "IP_ADDRESS":
+            if secret["tag"] == "IP_ADDRESS":  # ruff: noqa: SIM102
                 # if secret value in popular DNS servers, skip it
                 if is_private_ip(secret["value"]) or (
                     secret["value"] in POPULAR_DNS_SERVERS
@@ -146,10 +145,9 @@ def redact_pii_text(text, secrets, replacements, add_references=False):
     else:
         new_text = text
         references = ""
-    result = (
+    return (
         (new_text, references, modified) if add_references else (new_text, modified)
     )
-    return result
 
 
 def redact_pii(text, secrets, has_secrets, replacements):
@@ -160,5 +158,5 @@ def redact_pii(text, secrets, has_secrets, replacements):
     if has_secrets:
         new_text, _ = redact_pii_text(text, secrets, replacements)
         return new_text
-    else:
-        return text
+
+    return text