diff --git a/article/migrations/0013_article_article_license.py b/article/migrations/0013_article_article_license.py new file mode 100644 index 00000000..73c1f297 --- /dev/null +++ b/article/migrations/0013_article_article_license.py @@ -0,0 +1,52 @@ +# Generated by Django 5.0.3 on 2024-05-28 18:09 + +from django.db import migrations, models + +def transfer_license_statements_fk_to_article_license(apps, schema_editor): + Article = apps.get_model('article', 'Article') + + articles_to_update = [] + for instance in Article.objects.all(): + + new_license = None + if instance.license_statements.exists() and instance.license_statements.first().url: + new_license = instance.license_statements.first().url + elif instance.license and instance.license.license_type: + new_license = instance.license.license_type + + if new_license: + instance.article_license = new_license + articles_to_update.append(instance) + + if articles_to_update: + Article.objects.bulk_update(articles_to_update, ['article_license']) + + +def reverse_transfer_license_statements_fk_to_article_license(apps, schema_editor): + Article = apps.get_model('article', 'Article') + + articles_to_update = [] + for instance in Article.objects.all(): + instance.article_license = None + articles_to_update.append(instance) + + if articles_to_update: + Article.objects.bulk_update(articles_to_update, ['article_license']) + +class Migration(migrations.Migration): + + dependencies = [ + ("article", "0012_alter_article_publisher"), + ] + + operations = [ + migrations.AddField( + model_name="article", + name="article_license", + field=models.CharField(blank=True, max_length=255, null=True), + ), + migrations.RunPython( + transfer_license_statements_fk_to_article_license, + reverse_transfer_license_statements_fk_to_article_license, + ), + ] diff --git a/article/models.py b/article/models.py index 16d1541c..2fc6a72c 100755 --- a/article/models.py +++ b/article/models.py @@ -84,6 +84,7 @@ class Article(ExportModelOperationsMixin('article'), CommonControlField, Cluster license = models.ForeignKey( License, on_delete=models.SET_NULL, null=True, blank=True ) + article_license = models.CharField(max_length=255, null=True, blank=True) issue = models.ForeignKey(Issue, on_delete=models.SET_NULL, null=True, blank=True) first_page = models.CharField(max_length=20, null=True, blank=True) last_page = models.CharField(max_length=20, null=True, blank=True) diff --git a/article/sources/xmlsps.py b/article/sources/xmlsps.py index 88920254..8f5e4b15 100755 --- a/article/sources/xmlsps.py +++ b/article/sources/xmlsps.py @@ -115,14 +115,11 @@ def load_article(user, xml=None, file_path=None, v3=None): ) article.doi.set(get_or_create_doi(xmltree=xmltree, user=user)) - article.license_statements.set(get_licenses(xmltree=xmltree, user=user)) - for ls in article.license_statements.iterator(): - article.license = ls.license - article.save() - break + article.license_statements.set(set_license(xmltree=xmltree, article=article, user=user)) article.publisher = get_or_create_publisher(xmltree=xmltree, user=user, item=pid_v3) article.valid = True article.save() + logging.info(f"The article {pid_v3} has been processed") except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() xml_detail_error = etree.tostring(xmltree) @@ -258,36 +255,43 @@ def get_or_create_toc_sections(xmltree, user): return data -def get_licenses(xmltree, user): +def set_license(xmltree, article, user): xml_licenses = ArticleLicense(xmltree=xmltree).licenses - data = [] - license = None for xml_license in xml_licenses: - - if not license and xml_license.get("link"): - url_data = LicenseStatement.parse_url(xml_license.get("link")) - license_type = url_data.get("license_type") - if license_type: - license = License.create_or_update( - user=user, - license_type=license_type, - ) - if license or xml_license.get("license_p"): - try: - license_p = xml_license.get("license_p") - license_p = license_p["html_text"] - except (ValueError, TypeError, KeyError): - pass - - obj = LicenseStatement.create_or_update( - user=user, - url=xml_license.get("link"), - language=Language.get_or_create(code2=xml_license.get("lang")), - license_p=license_p, - license=license, - ) - data.append(obj) - return data + if license := xml_license.get("link"): + article.article_license = license + + +# def get_licenses(xmltree, user): +# xml_licenses = ArticleLicense(xmltree=xmltree).licenses +# data = [] +# license = None +# for xml_license in xml_licenses: + +# if not license and xml_license.get("link"): +# url_data = LicenseStatement.parse_url(xml_license.get("link")) +# license_type = url_data.get("license_type") +# if license_type: +# license = License.create_or_update( +# user=user, +# license_type=license_type, +# ) +# if license or xml_license.get("license_p"): +# try: +# license_p = xml_license.get("license_p") +# license_p = license_p["html_text"] +# except (ValueError, TypeError, KeyError): +# pass + +# obj = LicenseStatement.create_or_update( +# user=user, +# url=xml_license.get("link"), +# language=Language.get_or_create(code2=xml_license.get("lang")), +# license_p=license_p, +# license=license, +# ) +# data.append(obj) +# return data def create_or_update_keywords(xmltree, user, item): @@ -449,6 +453,7 @@ def get_or_create_institution_authors(xmltree, user, item): affiliation=affiliation, user=user, ) + data.append(obj) except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() UnexpectedEvent.create( diff --git a/article/tests.py b/article/tests.py index 7ce503c2..7886547b 100755 --- a/article/tests.py +++ b/article/tests.py @@ -1,3 +1,20 @@ from django.test import TestCase +from django_test_migrations.migrator import Migrator -# Create your tests here. +class TestArticleMigration(TestCase): + def test_migration_0013_article_article_license(self): + migrator = Migrator(database='default') + old_state = migrator.apply_initial_migration(('article', '0012_alter_article_publisher')) + Article = old_state.apps.get_model('article', 'Article') + LicenseStatement = old_state.apps.get_model('core', 'LicenseStatement') + article = Article.objects.create() + license_statement = LicenseStatement.objects.create(url="https://www.teste.com.br") + article.license_statements.add(license_statement) + + new_state = migrator.apply_tested_migration(('article', '0013_article_article_license')) + + Article = new_state.apps.get_model('article', 'Article') + + article = Article.objects.first() + self.assertEqual(article.article_license, 'https://www.teste.com.br') + migrator.reset() \ No newline at end of file diff --git a/config/settings/base.py b/config/settings/base.py index d6dea00e..6e7dc3a9 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -500,4 +500,4 @@ # Timeout function fetch_data -FETCH_DATA_TIMEOUT = env.str("FETCH_DATA_TIMEOUT", default=10) \ No newline at end of file +FETCH_DATA_TIMEOUT = env.int("FETCH_DATA_TIMEOUT", default=10) \ No newline at end of file diff --git a/core/migrations/0004_language_core_langua_code2_4f7261_idx.py b/core/migrations/0004_language_core_langua_code2_4f7261_idx.py new file mode 100644 index 00000000..dfd3ec70 --- /dev/null +++ b/core/migrations/0004_language_core_langua_code2_4f7261_idx.py @@ -0,0 +1,19 @@ +# Generated by Django 5.0.3 on 2024-05-27 17:45 + +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("core", "0003_gender_created_gender_creator_gender_updated_and_more"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.AddIndex( + model_name="language", + index=models.Index(fields=["code2"], name="core_langua_code2_4f7261_idx"), + ), + ] diff --git a/core/models.py b/core/models.py index 4a51da10..78f6b235 100755 --- a/core/models.py +++ b/core/models.py @@ -145,6 +145,13 @@ def autocomplete_label(self): class Meta: verbose_name = _("Language") verbose_name_plural = _("Languages") + indexes = [ + models.Index( + fields=[ + "code2", + ] + ), + ] def __unicode__(self): if self.name or self.code2: diff --git a/core/tests_standardizer.py b/core/tests_standardizer.py index 93377b24..42df4087 100644 --- a/core/tests_standardizer.py +++ b/core/tests_standardizer.py @@ -1,4 +1,4 @@ -from django.test import TestCase +from django.test import TestCase, SimpleTestCase # Create your tests here. from core.utils import standardizer @@ -65,3 +65,23 @@ def test_standardize_name(self): for i, item in enumerate(result): with self.subTest(i): self.assertEqual({"name": expected[i]}, item) + + +class StandardizerRemoveSpaceExtraTest(SimpleTestCase): + + def test_remove_extra_spaces(self): + test_cases = [ + (" Palavra1", "Palavra1"), + (" Palavra1 Palavra2 ", "Palavra1 Palavra2"), + ("", ""), + (" ", ""), + (" Palavra1 Palavra2 Palavra3 ", "Palavra1 Palavra2 Palavra3"), + (" Multiple spaces between words ", "Multiple spaces between words"), + ("\tTabs\tand\nnewlines\n", "Tabs and newlines"), + ] + + for text, expected in test_cases: + with self.subTest(text=text, expected=expected): + result = standardizer.remove_extra_spaces(text=text) + self.assertEqual(expected, result) + diff --git a/core/utils/standardizer.py b/core/utils/standardizer.py index a8f06914..4e9be771 100644 --- a/core/utils/standardizer.py +++ b/core/utils/standardizer.py @@ -9,11 +9,10 @@ def remove_extra_spaces(text): - text = text and text.strip() if not text: return text - # padroniza a quantidade de espaços - return " ".join([item.strip() for item in text.split() if item.strip()]) + # Padroniza a quantidade de espaços + return " ".join(text.split()) def standardize_code_and_name(original): diff --git a/location/models.py b/location/models.py index 0f6dba4c..a93e769c 100755 --- a/location/models.py +++ b/location/models.py @@ -457,7 +457,6 @@ def create_or_update( country_names[lang_code2] = name for lang_code2, text in country_names.items(): - logging.info(f"{lang_code2} {text}") language = Language.get_or_create(code2=lang_code2) CountryName.create_or_update( country=obj, language=language, text=text, user=user diff --git a/requirements/local.txt b/requirements/local.txt index 9cd031f0..84e76bf3 100644 --- a/requirements/local.txt +++ b/requirements/local.txt @@ -12,7 +12,7 @@ mypy==1.6.1 # https://github.com/python/mypy django-stubs==4.2.7 # https://github.com/typeddjango/django-stubs pytest==7.4.3 # https://github.com/pytest-dev/pytest pytest-sugar==0.9.7 # https://github.com/Frozenball/pytest-sugar - +django-test-migrations==1.3.0 # Documentation # ------------------------------------------------------------------------------ sphinx==7.2.6 # https://github.com/sphinx-doc/sphinx diff --git a/researcher/models.py b/researcher/models.py index ac48feb2..0444bb56 100644 --- a/researcher/models.py +++ b/researcher/models.py @@ -322,11 +322,7 @@ def get_full_name(self): @staticmethod def join_names(given_names, last_name, suffix): return " ".join( - [ - remove_extra_spaces(item) - for item in (given_names, last_name, suffix) - if remove_extra_spaces(item) - ] + filter(None, [given_names, last_name, suffix]) ) @classmethod @@ -380,6 +376,7 @@ def _create( obj.gender = gender obj.gender_identification_status = gender_identification_status obj.save() + return obj except IntegrityError: return cls._get(given_names, last_name, suffix, fullname, declared_name) except Exception as e: @@ -408,8 +405,7 @@ def get_or_create( given_names = remove_extra_spaces(given_names) last_name = remove_extra_spaces(last_name) suffix = remove_extra_spaces(suffix) - fullname = remove_extra_spaces(fullname) - fullname = fullname or PersonName.join_names(given_names, last_name, suffix) + fullname = remove_extra_spaces(fullname) or PersonName.join_names(given_names, last_name, suffix) try: return cls._get(given_names, last_name, suffix, fullname, declared_name) diff --git a/researcher/tests.py b/researcher/tests.py index 7ce503c2..111e624b 100644 --- a/researcher/tests.py +++ b/researcher/tests.py @@ -1,3 +1,18 @@ -from django.test import TestCase +from django.test import SimpleTestCase # Create your tests here. +from researcher.models import PersonName + + +class PersonNameJoinNameTest(SimpleTestCase): + def test_person_name_join_name(self): + test_cases = [ + (['Palavra1', None, None], 'Palavra1'), + (['Palavra1', 'Palavra2', None], 'Palavra1 Palavra2' ), + (['Palavra1', 'Palavra2', 'Palavra3'], 'Palavra1 Palavra2 Palavra3'), + ] + + for text, expected in test_cases: + with self.subTest(text=text, excepted=expected): + result = PersonName.join_names(*text) + self.assertEqual(expected, result) \ No newline at end of file