Fix language query to fetch only based on codename (#98)

* Fix black formatting

* Query language only by primary key + update values if not matching

backend/src/backend/exporters/kiwix_public_exporter.py

@@ -13,7 +13,6 @@ class KiwixPublicExporter(ExporterInterface):
     """Upload XML Kiwix Library to `UPLOAD_URI`, using `PRIVATE_KEY"""
 
     async def export():
-
         src_path = pathlib.Path(tempfile.NamedTemporaryFile(delete=False).name)
 
         try:

backend/src/backend/formatters/__init__.py

@@ -1,5 +1,4 @@
 class FormaterInterface:
-
     slug: str
     description: str
 

backend/src/backend/formatters/kiwixlibraryxml.py

@@ -9,7 +9,6 @@
 
 
 class KiwixLibraryXml(FormaterInterface):
-
     description: str = "Kiwix Library XML"
     slug = "kiwixlibraryxml"
 

backend/src/backend/models.py

@@ -8,7 +8,7 @@
 import pydantic
 import sqlalchemy
 
-from backend.constants import BackendConf
+from backend.constants import BackendConf, logger
 
 METADATA_MAX_LEN = 2048
 KIND_TEXT: str = "text"
@@ -75,6 +75,20 @@ class Meta(BaseMeta):
     def __repr__(self):
         return f"Language(code={self.code}, name={self.name})"
 
+    async def get_create_or_update(code: str, name: str, native: str):
+        language, created = await Language.objects.get_or_create(
+            code=code, _defaults={"name": name, "native": native}
+        )
+        if created is False and (language.name != name or language.native != native):
+            logger.warning(
+                f"Updating language values for {language.code} from {language.name}/"
+                f"{language.native} to {name}/{native}"
+            )
+            language.name = name
+            language.native = native
+            await language.update()
+        return language
+
 
 class TagMixin:
     """ZIM Tag is a text slug stored along others in `Tags` metadata

backend/src/backend/routes/books.py

@@ -126,7 +126,7 @@ async def create_book(book_payload: BookAddSchema):
 
     for lang_code in book_payload.metadata["Language"].split(","):
         native_name, english_name = find_language_names(lang_code)
-        language, _ = await Language.objects.get_or_create(
+        language = await Language.get_create_or_update(
             code=lang_code, name=english_name, native=native_name
         )
         await book.languages.add(language)

backend/src/backend/schemas.py

@@ -117,7 +117,6 @@ def get_for(
         with_tags: bool = False,
         with_metadata: Union[List[str], bool] = False,
     ):
-
         return getattr(
             cls,
             cls._name_for(
@@ -129,7 +128,6 @@ def get_for(
 
 
 class TitleSendSchema(BaseModel):
-
     ident: str
     languages: List[str]
     tags: List[str]

backend/src/demo/data.py

@@ -31,7 +31,6 @@ async def inner():
 
 @transaction
 async def load_fixture():
-
     lang_codes = [
         "lg",
         "hi",
@@ -154,7 +153,7 @@ async def load_fixture():
 
         iso_639_3_lang_code = get_language_details(lang_code)["iso-639-3"]
 
-        language, _ = await Language.objects.get_or_create(
+        language = await Language.get_create_or_update(
             code=iso_639_3_lang_code, name=english_name, native=native_name
         )
         await book.languages.add(language)

backend/tests/conftest.py

@@ -157,7 +157,6 @@ def book_dict(base64_png):
 
 @pytest.fixture(scope="function")
 async def title_without_book(book_dict):
-
     title = await Title.objects.create(ident="wikipedia_ar_mathematics")
 
     yield title
@@ -166,7 +165,6 @@ async def title_without_book(book_dict):
 
 @pytest.fixture(scope="function")
 async def book_dict_with_logs(book_dict):
-
     book_dict["zimcheck"].update(
         {
             "logs": [
@@ -510,7 +508,6 @@ async def titles_with_metadata(language_fra, language_eng):
 @pytest.fixture(scope="function")
 @pytest.mark.asyncio
 async def titles_with_metadata_books(client, book_dict):
-
     ids = []
     book_dict["metadata"]["Name"] = "wikipedia_en_all"
     book_dict["metadata"]["Flavour"] = "maxi"
@@ -534,7 +531,6 @@ async def titles_with_metadata_books(client, book_dict):
 @pytest.fixture(scope="function")
 @pytest.mark.asyncio
 async def title_with_three_books(client, book_dict):
-
     title = await Title.objects.create(ident="wikipedia_ar_mathematics")
     ids = []
     for period in ["2021-04-08", "2021-05-08", "2021-06-08"]:

backend/tests/test_books_endpoint.py

@@ -13,14 +13,15 @@
     KIND_TEXT,
     Book,
     BookMetadata,
+    Language,
     Title,
     TitleMetadata,
 )
 from backend.utils import get_ident_from_name
 
 
 @pytest.mark.asyncio
-async def test_add_book(client, book_dict, clear_book_dict):
+async def test_add_book_create_language(client, book_dict, clear_book_dict):
     response = await client.post("/v1/books/add", json=book_dict)
     assert response.status_code == 201
     assert response.headers.get("Content-Type") == "application/json"
@@ -39,8 +40,22 @@ async def test_add_book(client, book_dict, clear_book_dict):
 
 
 @pytest.mark.asyncio
-async def test_of_rollback(client, book_dict):
+async def test_add_book_update_language(
+    client, book_dict, clear_book_dict, language_eng
+):
+    # language_eng is already created in DB but its name and native properties are wrong
+    # since they have been updated in Babel
+    response = await client.post("/v1/books/add", json=book_dict)
+    assert response.status_code == 201
+    assert response.headers.get("Content-Type") == "application/json"
 
+    language = await Language.objects.get(code="eng")
+    assert language.native == "English (United States)"
+    assert language.name == "English (United States)"
+
+
+@pytest.mark.asyncio
+async def test_of_rollback(client, book_dict):
     ident = get_ident_from_name(book_dict["metadata"]["Name"])
     book_dict["metadata"]["Language"] = ""
     response = await client.post("/v1/books/add", json=book_dict)

backend/tests/test_collections.py

@@ -7,7 +7,6 @@
 
 @pytest.mark.asyncio
 async def test_collection_interface():
-
     with pytest.raises(NotImplementedError):
         for _ in await CollectionInterface().get_book_ids():
             ...
@@ -19,7 +18,6 @@ async def test_collection_interface():
 
 @pytest.mark.asyncio
 async def test_kiwix_public(titles_with_metadata_books):
-
     books_ids = KiwixPublicCollection().get_book_ids()
     async for book_id in books_ids:
         assert await Book.objects.filter(id=book_id).exists()

backend/tests/test_exporter.py

@@ -10,7 +10,6 @@
 
 @pytest.mark.asyncio
 async def test_exporter_interface():
-
     with pytest.raises(NotImplementedError):
         for _ in await ExporterInterface().export():
             ...

backend/tests/test_remove_obsolete_books.py

@@ -7,7 +7,6 @@
 
 @pytest.mark.asyncio
 async def test_remove_obsolete_books(client, book_dict, title_with_three_books):
-
     title = (
         await Title.objects.exclude_fields(["tags", "languages"])
         .select_related("books")

backend/tests/test_zimcheck_dashboard.py

@@ -3,7 +3,6 @@
 
 @pytest.mark.asyncio
 async def test_zim_check_dashboard(client, book_dict_with_logs, title_without_book):
-
     response = await client.get("/v1/zimcheck")
     assert response.status_code == 200
     assert response.headers.get("Content-Type") == "application/json"