fix the use of BPEmb in download models

GRAAL-Research · Jul 9, 2024 · e44a7ce · e44a7ce
1 parent bd04738
commit e44a7ce
Show file tree

Hide file tree

Showing 9 changed files with 35 additions and 23 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -356,4 +356,8 @@
 
 - Fix Sentry version error in Docker Image.
 
+## 0.9.12
+
+- Bug-fix the call to the BPEmb class instead of the BPEmbBaseURLWrapperBugFix to fix the download URL in `download_models`.
+
 ## dev
diff --git a/deepparse/__init__.py b/deepparse/__init__.py
@@ -1,4 +1,5 @@
 # pylint: disable=wildcard-import
+from .bpemb_url_bug_fix import *
 from .download_tools import *
 from .validations import *
 from .version import __version__

diff --git a/deepparse/bpemb_url_bug_fix.py b/deepparse/bpemb_url_bug_fix.py
@@ -0,0 +1,13 @@
+"""
+Due to an error in the BPEmb base URL to download the weights, and since the authors and maintainer do not respond or
+seems to maintain the project; we use a wrapper to bug-fix the URL to change it.
+However, the wrapper must be placed here due to circular import.
+"""
+
+from bpemb import BPEmb
+
+
+class BPEmbBaseURLWrapperBugFix(BPEmb):
+    def __init__(self, **kwargs):
+        self.base_url = "https://bpemb.h-its.org/multi/"
+        super().__init__(**kwargs)
diff --git a/deepparse/download_tools.py b/deepparse/download_tools.py
@@ -12,7 +12,7 @@
 from requests import HTTPError
 from urllib3.exceptions import MaxRetryError
 
-from .embeddings_models import BPEmbBaseURLWrapperBugFix
+from .bpemb_url_bug_fix import BPEmbBaseURLWrapperBugFix
 from .errors.server_error import ServerError
 
 BASE_URL = "https://graal.ift.ulaval.ca/public/deepparse/{}.{}"

diff --git a/deepparse/embeddings_models/bpemb_embeddings_model.py b/deepparse/embeddings_models/bpemb_embeddings_model.py
@@ -3,19 +3,13 @@
 from pathlib import Path
 
 import requests
-from bpemb import BPEmb
 from numpy.core.multiarray import ndarray
 from urllib3.exceptions import InsecureRequestWarning
 
+from ..bpemb_url_bug_fix import BPEmbBaseURLWrapperBugFix
 from .embeddings_model import EmbeddingsModel
 
 
-class BPEmbBaseURLWrapperBugFix(BPEmb):
-    def __init__(self, **kwargs):
-        self.base_url = "https://bpemb.h-its.org/multi/"
-        super().__init__(**kwargs)
-
-
 class BPEmbEmbeddingsModel(EmbeddingsModel):
     """
     BPEmb embeddings network from `BPEmb: Tokenization-free Pre-trained Subword Embeddings in 275 Languages

diff --git a/tests/cli/test_download_model.py b/tests/cli/test_download_model.py
@@ -60,7 +60,7 @@ def test_givenAFasttextMagnitudeDownload_whenModelIsNotLocal_thenDownloadEmbeddi
     @patch("deepparse.download_tools.download_weights")
     def test_givenABPembDownload_whenModelIsNotLocal_thenDownloadEmbeddings(self, weights_download_mock):
         with patch("deepparse.download_tools.CACHE_PATH", self.fake_cache_dir):
-            with patch("deepparse.download_tools.BPEmb") as downloader:
+            with patch("deepparse.download_tools.BPEmbBaseURLWrapperBugFix") as downloader:
                 download_model_cli_main([self.a_bpemb_model_type])
 
                 downloader.assert_called()
@@ -95,7 +95,7 @@ def test_givenAFasttextLightDownload_whenModelIsNotLocal_thenDownloadWeights(sel
                 downloader.assert_called()
                 downloader.assert_any_call(self.a_fasttext_light_model_file_name, saving_dir=self.fake_cache_dir)
 
-    @patch("deepparse.download_tools.BPEmb")
+    @patch("deepparse.download_tools.BPEmbBaseURLWrapperBugFix")
     def test_givenABPembDownload_whenModelIsNotLocal_thenDownloadWeights(self, download_embeddings_mock):
         with patch("deepparse.download_tools.CACHE_PATH", self.fake_cache_dir):
             with patch("deepparse.download_tools.download_weights") as downloader:
@@ -104,7 +104,7 @@ def test_givenABPembDownload_whenModelIsNotLocal_thenDownloadWeights(self, downl
                 downloader.assert_called()
                 downloader.assert_any_call(self.a_bpemb_model_type, saving_dir=self.fake_cache_dir)
 
-    @patch("deepparse.download_tools.BPEmb")
+    @patch("deepparse.download_tools.BPEmbBaseURLWrapperBugFix")
     def test_givenABPembAttDownload_whenModelIsNotLocal_thenDownloadWeights(self, download_embeddings_mock):
         with patch("deepparse.download_tools.CACHE_PATH", self.fake_cache_dir):
             with patch("deepparse.download_tools.download_weights") as downloader:
@@ -139,7 +139,7 @@ def test_givenAFasttextLightDownload_whenModelIsLocalButNotLatest_thenDownloadWe
                 downloader.assert_called()
                 downloader.assert_any_call(self.a_fasttext_light_model_file_name, saving_dir=self.fake_cache_dir)
 
-    @patch("deepparse.download_tools.BPEmb")
+    @patch("deepparse.download_tools.BPEmbBaseURLWrapperBugFix")
     @patch("deepparse.download_tools.os.path.isfile", return_value=True)
     @patch("deepparse.download_tools.latest_version", return_value=False)  # not the latest version
     def test_givenABPembDownload_whenModelIsLocalButNotLatest_thenDownloadWeights(
@@ -179,7 +179,7 @@ def test_givenAFasttextLightDownload_whenModelIsLocalAndGoodVersion_thenDoNoting
 
                 downloader.assert_not_called()
 
-    @patch("deepparse.download_tools.BPEmb")
+    @patch("deepparse.download_tools.BPEmbBaseURLWrapperBugFix")
     @patch("deepparse.download_tools.os.path.isfile", return_value=True)
     @patch("deepparse.download_tools.latest_version", return_value=True)  # the latest version
     def test_givenABPembDownload_whenModelIsLocalAndGoodVersion_thenDoNoting(
@@ -218,7 +218,7 @@ def test_givenAFasttextLightDownload_whenModelIsNotLocalButNotLatest_thenDownloa
                 downloader.assert_called()
                 downloader.assert_any_call(self.a_fasttext_light_model_file_name, saving_dir=self.fake_cache_dir)
 
-    @patch("deepparse.download_tools.BPEmb")
+    @patch("deepparse.download_tools.BPEmbBaseURLWrapperBugFix")
     @patch("deepparse.download_tools.os.path.isfile", side_effect=[False, True])  # no version file in local
     def test_givenABPembDownload_whenModelIsNotLocalButNotLatest_thenDownloadWeights(
         self, download_embeddings_mock, os_is_file_mock

diff --git a/tests/cli/test_download_models.py b/tests/cli/test_download_models.py
@@ -31,7 +31,7 @@ def tearDown(self) -> None:
     @patch("deepparse.download_tools.download_weights")
     @patch("deepparse.download_tools.download_fasttext_embeddings")
     @patch("deepparse.download_tools.download_fasttext_magnitude_embeddings")
-    @patch("deepparse.download_tools.BPEmb")
+    @patch("deepparse.download_tools.BPEmbBaseURLWrapperBugFix")
     def test_givenADownloadAllModels_whenModelsAreNotLocal_thenDownloadAllModels(
         self,
         BPEmb_mock,

diff --git a/tests/test_download_tools.py b/tests/test_download_tools.py
@@ -352,7 +352,7 @@ def tearDown(self) -> None:
     @patch("deepparse.download_tools.download_weights")
     @patch("deepparse.download_tools.download_fasttext_embeddings")
     @patch("deepparse.download_tools.download_fasttext_magnitude_embeddings")
-    @patch("deepparse.download_tools.BPEmb")
+    @patch("deepparse.download_tools.BPEmbBaseURLWrapperBugFix")
     def test_givenADownloadAllModels_whenModelsAreNotLocal_thenDownloadAllModels(
         self,
         BPEmb_mock,
@@ -428,7 +428,7 @@ def test_givenAFasttextMagnitudeDownload_whenModelIsNotLocal_thenDownloadEmbeddi
     @patch("deepparse.download_tools.download_weights")
     def test_givenABPembDownload_whenModelIsNotLocal_thenDownloadEmbeddings(self, weights_download_mock):
         with patch("deepparse.download_tools.CACHE_PATH", self.fake_cache_dir):
-            with patch("deepparse.download_tools.BPEmb") as downloader:
+            with patch("deepparse.download_tools.BPEmbBaseURLWrapperBugFix") as downloader:
                 download_model(self.a_bpemb_model_type)
 
                 downloader.assert_called()
@@ -463,7 +463,7 @@ def test_givenAFasttextLightDownload_whenModelIsNotLocal_thenDownloadWeights(sel
                 downloader.assert_called()
                 downloader.assert_any_call(self.a_fasttext_light_model_file_name, saving_dir=self.fake_cache_dir)
 
-    @patch("deepparse.download_tools.BPEmb")
+    @patch("deepparse.download_tools.BPEmbBaseURLWrapperBugFix")
     def test_givenABPembDownload_whenModelIsNotLocal_thenDownloadWeights(self, download_embeddings_mock):
         with patch("deepparse.download_tools.CACHE_PATH", self.fake_cache_dir):
             with patch("deepparse.download_tools.download_weights") as downloader:
@@ -472,7 +472,7 @@ def test_givenABPembDownload_whenModelIsNotLocal_thenDownloadWeights(self, downl
                 downloader.assert_called()
                 downloader.assert_any_call(self.a_bpemb_model_type, saving_dir=self.fake_cache_dir)
 
-    @patch("deepparse.download_tools.BPEmb")
+    @patch("deepparse.download_tools.BPEmbBaseURLWrapperBugFix")
     def test_givenABPembAttDownload_whenModelIsNotLocal_thenDownloadWeights(self, download_embeddings_mock):
         with patch("deepparse.download_tools.CACHE_PATH", self.fake_cache_dir):
             with patch("deepparse.download_tools.download_weights") as downloader:
@@ -507,7 +507,7 @@ def test_givenAFasttextLightDownload_whenModelIsLocalButNotLatest_thenDownloadWe
                 downloader.assert_called()
                 downloader.assert_any_call(self.a_fasttext_light_model_file_name, saving_dir=self.fake_cache_dir)
 
-    @patch("deepparse.download_tools.BPEmb")
+    @patch("deepparse.download_tools.BPEmbBaseURLWrapperBugFix")
     @patch("deepparse.download_tools.os.path.isfile", return_value=True)
     @patch("deepparse.download_tools.latest_version", return_value=False)  # not the latest version
     def test_givenABPembDownload_whenModelIsLocalButNotLatest_thenDownloadWeights(
@@ -547,7 +547,7 @@ def test_givenAFasttextLightDownload_whenModelIsLocalAndGoodVersion_thenDoNoting
 
                 downloader.assert_not_called()
 
-    @patch("deepparse.download_tools.BPEmb")
+    @patch("deepparse.download_tools.BPEmbBaseURLWrapperBugFix")
     @patch("deepparse.download_tools.os.path.isfile", return_value=True)
     @patch("deepparse.download_tools.latest_version", return_value=True)  # the latest version
     def test_givenABPembDownload_whenModelIsLocalAndGoodVersion_thenDoNoting(
@@ -586,7 +586,7 @@ def test_givenAFasttextLightDownload_whenModelIsNotLocalButNotLatest_thenDownloa
                 downloader.assert_called()
                 downloader.assert_any_call(self.a_fasttext_light_model_file_name, saving_dir=self.fake_cache_dir)
 
-    @patch("deepparse.download_tools.BPEmb")
+    @patch("deepparse.download_tools.BPEmbBaseURLWrapperBugFix")
     @patch("deepparse.download_tools.os.path.isfile", side_effect=[False, True])  # no version file in local
     def test_givenABPembDownload_whenModelIsNotLocalButNotLatest_thenDownloadWeights(
         self, download_embeddings_mock, os_is_file_mock

diff --git a/version.txt b/version.txt
@@ -1 +1 @@
-0.9.11
+0.9.12