Merge pull request #758 from samuelveigarangel/feature/article-license

Adiciona um novo campo em Article para representar licença do artigo.

article/migrations/0013_article_article_license.py

@@ -0,0 +1,52 @@
+# Generated by Django 5.0.3 on 2024-05-28 18:09
+
+from django.db import migrations, models
+
+def transfer_license_statements_fk_to_article_license(apps, schema_editor):
+    Article = apps.get_model('article', 'Article')
+
+    articles_to_update = []
+    for instance in Article.objects.all():
+
+        new_license = None
+        if instance.license_statements.exists() and instance.license_statements.first().url:
+            new_license = instance.license_statements.first().url
+        elif instance.license and instance.license.license_type:
+            new_license = instance.license.license_type
+
+        if new_license:
+            instance.article_license = new_license
+            articles_to_update.append(instance)
+
+    if articles_to_update:
+        Article.objects.bulk_update(articles_to_update, ['article_license'])
+
+
+def reverse_transfer_license_statements_fk_to_article_license(apps, schema_editor):
+    Article = apps.get_model('article', 'Article')
+
+    articles_to_update = []
+    for instance in Article.objects.all():
+        instance.article_license = None
+        articles_to_update.append(instance)
+
+    if articles_to_update:
+        Article.objects.bulk_update(articles_to_update, ['article_license'])
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("article", "0012_alter_article_publisher"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="article",
+            name="article_license",
+            field=models.CharField(blank=True, max_length=255, null=True),
+        ),
+        migrations.RunPython(
+            transfer_license_statements_fk_to_article_license,
+            reverse_transfer_license_statements_fk_to_article_license,
+            ),
+    ]

article/models.py

@@ -84,6 +84,7 @@ class Article(ExportModelOperationsMixin('article'), CommonControlField, Cluster
     license = models.ForeignKey(
         License, on_delete=models.SET_NULL, null=True, blank=True
     )
+    article_license = models.CharField(max_length=255, null=True, blank=True)
     issue = models.ForeignKey(Issue, on_delete=models.SET_NULL, null=True, blank=True)
     first_page = models.CharField(max_length=20, null=True, blank=True)
     last_page = models.CharField(max_length=20, null=True, blank=True)

article/sources/xmlsps.py

@@ -115,14 +115,11 @@ def load_article(user, xml=None, file_path=None, v3=None):
         )
         article.doi.set(get_or_create_doi(xmltree=xmltree, user=user))
 
-        article.license_statements.set(get_licenses(xmltree=xmltree, user=user))
-        for ls in article.license_statements.iterator():
-            article.license = ls.license
-            article.save()
-            break
+        article.license_statements.set(set_license(xmltree=xmltree, article=article, user=user))
         article.publisher = get_or_create_publisher(xmltree=xmltree, user=user, item=pid_v3)
         article.valid = True
         article.save()
+        logging.info(f"The article {pid_v3} has been processed")
     except Exception as e:
         exc_type, exc_value, exc_traceback = sys.exc_info()
         xml_detail_error = etree.tostring(xmltree)
@@ -258,36 +255,43 @@ def get_or_create_toc_sections(xmltree, user):
     return data
 
 
-def get_licenses(xmltree, user):
+def set_license(xmltree, article, user):
     xml_licenses = ArticleLicense(xmltree=xmltree).licenses
-    data = []
-    license = None
     for xml_license in xml_licenses:
-
-        if not license and xml_license.get("link"):
-            url_data = LicenseStatement.parse_url(xml_license.get("link"))
-            license_type = url_data.get("license_type")
-            if license_type:
-                license = License.create_or_update(
-                    user=user,
-                    license_type=license_type,
-                )
-        if license or xml_license.get("license_p"):
-            try:
-                license_p = xml_license.get("license_p")
-                license_p = license_p["html_text"]
-            except (ValueError, TypeError, KeyError):
-                pass
-
-            obj = LicenseStatement.create_or_update(
-                user=user,
-                url=xml_license.get("link"),
-                language=Language.get_or_create(code2=xml_license.get("lang")),
-                license_p=license_p,
-                license=license,
-            )
-            data.append(obj)
-    return data
+        if license := xml_license.get("link"):
+            article.article_license = license
+
+
+# def get_licenses(xmltree, user):
+#     xml_licenses = ArticleLicense(xmltree=xmltree).licenses
+#     data = []
+#     license = None
+#     for xml_license in xml_licenses:
+
+#         if not license and xml_license.get("link"):
+#             url_data = LicenseStatement.parse_url(xml_license.get("link"))
+#             license_type = url_data.get("license_type")
+#             if license_type:
+#                 license = License.create_or_update(
+#                     user=user,
+#                     license_type=license_type,
+#                 )
+#         if license or xml_license.get("license_p"):
+#             try:
+#                 license_p = xml_license.get("license_p")
+#                 license_p = license_p["html_text"]
+#             except (ValueError, TypeError, KeyError):
+#                 pass
+
+#             obj = LicenseStatement.create_or_update(
+#                 user=user,
+#                 url=xml_license.get("link"),
+#                 language=Language.get_or_create(code2=xml_license.get("lang")),
+#                 license_p=license_p,
+#                 license=license,
+#             )
+#             data.append(obj)
+#     return data
 
 
 def create_or_update_keywords(xmltree, user, item):
@@ -449,6 +453,7 @@ def get_or_create_institution_authors(xmltree, user, item):
                     affiliation=affiliation,
                     user=user,
                 )
+                data.append(obj)
         except Exception as e:
             exc_type, exc_value, exc_traceback = sys.exc_info()
             UnexpectedEvent.create(

article/tests.py

@@ -1,3 +1,20 @@
 from django.test import TestCase
+from django_test_migrations.migrator import Migrator
 
-# Create your tests here.
+class TestArticleMigration(TestCase):
+    def test_migration_0013_article_article_license(self):
+        migrator = Migrator(database='default')
+        old_state = migrator.apply_initial_migration(('article', '0012_alter_article_publisher'))
+        Article = old_state.apps.get_model('article', 'Article')
+        LicenseStatement = old_state.apps.get_model('core', 'LicenseStatement')
+        article = Article.objects.create()
+        license_statement = LicenseStatement.objects.create(url="https://www.teste.com.br")
+        article.license_statements.add(license_statement)
+
+        new_state = migrator.apply_tested_migration(('article', '0013_article_article_license'))
+
+        Article = new_state.apps.get_model('article', 'Article')
+
+        article = Article.objects.first()
+        self.assertEqual(article.article_license, 'https://www.teste.com.br')
+        migrator.reset()

config/settings/base.py

@@ -500,4 +500,4 @@
 
 
 # Timeout function fetch_data
-FETCH_DATA_TIMEOUT = env.str("FETCH_DATA_TIMEOUT", default=10)
+FETCH_DATA_TIMEOUT = env.int("FETCH_DATA_TIMEOUT", default=10)

core/migrations/0004_language_core_langua_code2_4f7261_idx.py

@@ -0,0 +1,19 @@
+# Generated by Django 5.0.3 on 2024-05-27 17:45
+
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("core", "0003_gender_created_gender_creator_gender_updated_and_more"),
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+    ]
+
+    operations = [
+        migrations.AddIndex(
+            model_name="language",
+            index=models.Index(fields=["code2"], name="core_langua_code2_4f7261_idx"),
+        ),
+    ]

core/models.py

@@ -145,6 +145,13 @@ def autocomplete_label(self):
     class Meta:
         verbose_name = _("Language")
         verbose_name_plural = _("Languages")
+        indexes = [
+            models.Index(
+                fields=[
+                    "code2",
+                ]
+            ),
+        ]
 
     def __unicode__(self):
         if self.name or self.code2:

core/tests_standardizer.py

@@ -1,4 +1,4 @@
-from django.test import TestCase
+from django.test import TestCase, SimpleTestCase
 
 # Create your tests here.
 from core.utils import standardizer
@@ -65,3 +65,23 @@ def test_standardize_name(self):
         for i, item in enumerate(result):
             with self.subTest(i):
                 self.assertEqual({"name": expected[i]}, item)
+
+
+class StandardizerRemoveSpaceExtraTest(SimpleTestCase):
+
+    def test_remove_extra_spaces(self):
+        test_cases = [
+            ("  Palavra1", "Palavra1"),
+            ("    Palavra1      Palavra2   ", "Palavra1 Palavra2"),
+            ("", ""),
+            ("   ", ""),
+            (" Palavra1   Palavra2 Palavra3 ", "Palavra1 Palavra2 Palavra3"),
+            ("   Multiple   spaces   between  words   ", "Multiple spaces between words"),
+            ("\tTabs\tand\nnewlines\n", "Tabs and newlines"),
+        ]
+
+        for text, expected in test_cases:
+            with self.subTest(text=text, expected=expected):
+                result = standardizer.remove_extra_spaces(text=text)
+                self.assertEqual(expected, result)
+

core/utils/standardizer.py

@@ -9,11 +9,10 @@
 
 
 def remove_extra_spaces(text):
-    text = text and text.strip()
     if not text:
         return text
-    # padroniza a quantidade de espaços
-    return " ".join([item.strip() for item in text.split() if item.strip()])
+    # Padroniza a quantidade de espaços
+    return " ".join(text.split())
 
 
 def standardize_code_and_name(original):

location/models.py

@@ -457,7 +457,6 @@ def create_or_update(
             country_names[lang_code2] = name
 
         for lang_code2, text in country_names.items():
-            logging.info(f"{lang_code2} {text}")
             language = Language.get_or_create(code2=lang_code2)
             CountryName.create_or_update(
                 country=obj, language=language, text=text, user=user

requirements/local.txt

@@ -12,7 +12,7 @@ mypy==1.6.1  # https://github.com/python/mypy
 django-stubs==4.2.7  # https://github.com/typeddjango/django-stubs
 pytest==7.4.3  # https://github.com/pytest-dev/pytest
 pytest-sugar==0.9.7  # https://github.com/Frozenball/pytest-sugar
-
+django-test-migrations==1.3.0
 # Documentation
 # ------------------------------------------------------------------------------
 sphinx==7.2.6  # https://github.com/sphinx-doc/sphinx

researcher/models.py

@@ -322,11 +322,7 @@ def get_full_name(self):
     @staticmethod
     def join_names(given_names, last_name, suffix):
         return " ".join(
-            [
-                remove_extra_spaces(item)
-                for item in (given_names, last_name, suffix)
-                if remove_extra_spaces(item)
-            ]
+            filter(None, [given_names, last_name, suffix])
         )
 
     @classmethod
@@ -380,6 +376,7 @@ def _create(
             obj.gender = gender
             obj.gender_identification_status = gender_identification_status
             obj.save()
+            return obj
         except IntegrityError:
             return cls._get(given_names, last_name, suffix, fullname, declared_name)
         except Exception as e:
@@ -408,8 +405,7 @@ def get_or_create(
         given_names = remove_extra_spaces(given_names)
         last_name = remove_extra_spaces(last_name)
         suffix = remove_extra_spaces(suffix)
-        fullname = remove_extra_spaces(fullname)
-        fullname = fullname or PersonName.join_names(given_names, last_name, suffix)
+        fullname = remove_extra_spaces(fullname) or PersonName.join_names(given_names, last_name, suffix)
 
         try:
             return cls._get(given_names, last_name, suffix, fullname, declared_name)

researcher/tests.py

@@ -1,3 +1,18 @@
-from django.test import TestCase
+from django.test import SimpleTestCase
 
 # Create your tests here.
+from researcher.models import PersonName
+
+
+class PersonNameJoinNameTest(SimpleTestCase):
+    def test_person_name_join_name(self):
+        test_cases = [
+            (['Palavra1', None, None], 'Palavra1'),
+            (['Palavra1', 'Palavra2', None], 'Palavra1 Palavra2' ),
+            (['Palavra1', 'Palavra2', 'Palavra3'], 'Palavra1 Palavra2 Palavra3'),
+        ]
+
+        for text, expected in test_cases:
+            with self.subTest(text=text, excepted=expected):
+                result = PersonName.join_names(*text)
+                self.assertEqual(expected, result)