feat: changed configs & pyproject

huggingface · Sep 27, 2024 · ea3a915 · ea3a915
1 parent c085736
commit ea3a915
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 0 deletions.
diff --git a/pyproject.toml b/pyproject.toml
@@ -55,6 +55,9 @@ processing = [
 #   "readability-lxml @ git+https://github.com/huggingface/python-readability.git@speedup",
     "tldextract",
     "trafilatura>=1.8.0,<1.12.0",
+    "justext",
+    "resiliparse",
+    "readabilipy",
     "tokenizers",
     "ftfy",
     "fasteners",

diff --git a/src/datatrove/pipeline/extractors/__init__.py b/src/datatrove/pipeline/extractors/__init__.py
@@ -1,4 +1,7 @@
+from .inscriptis import Inscriptis
 from .justext import Justext
 from .modular import ReadabilityInscriptis
+from .readabilipy import ReadabiliPy
+from .readability import Readability
 from .resiliparse import Resiliparse
 from .trafilatura import Trafilatura