diff --git a/arches_lingo/const.py b/arches_lingo/const.py index 4cdf3bb0..776a52d3 100644 --- a/arches_lingo/const.py +++ b/arches_lingo/const.py @@ -4,7 +4,6 @@ TOP_CONCEPT_OF_NODE_AND_NODEGROUP = "bf73e5b9-4888-11ee-8a8d-11afefc4bff7" # classification_status -BROADER_NODE_AND_NODEGROUP = "f3f7bbea-0eb9-11ef-93db-0a58a9feac02" CLASSIFICATION_STATUS_NODEGROUP = "f3f7bbea-0eb9-11ef-93db-0a58a9feac02" CLASSIFICATION_STATUS_ASCRIBED_CLASSIFICATION_NODEID = ( "0b531a82-0eba-11ef-93db-0a58a9feac02" @@ -27,7 +26,7 @@ # appellative_status CONCEPT_NAME_NODEGROUP = "ab9fee9c-0eb6-11ef-93db-0a58a9feac02" # appellative_status_ascribed_name_content -CONCEPT_NAME_CONTENT_NODE = "bf73e695-4888-11ee-8a8d-11afefc4bff7" +CONCEPT_NAME_CONTENT_NODE = "7357993a-0eb7-11ef-93db-0a58a9feac02" # appellative_status_ascribed_name_language CONCEPT_NAME_LANGUAGE_NODE = "a8ecaf54-0eb7-11ef-93db-0a58a9feac02" # appellative_status_ascribed_relation @@ -49,8 +48,9 @@ SCHEME_NAME_TYPE_NODE = "ef87b588-11de-11ef-9493-0a58a9feac02" -PREF_LABEL_VALUE_ID = "3b8a03f1-9047-48e4-9ca0-b3fe887f6f9d" -ALT_LABEL_VALUE_ID = "c02f97c5-da16-4ff0-864a-92c34da84e38" +PREF_LABEL_VALUE_ID = "6ac8e471-476e-4fd0-b276-86e01a17bcc8" +ALT_LABEL_VALUE_ID = "d9e362ca-c155-4569-9197-5583dd66f7e5" +HIDDEN_LABEL_VALUE_ID = "18c46580-8c3c-48b7-9a6c-a0643708cb8b" # Old RDM concepts, values LANGUAGE_CONCEPT_ID = "845cc417-ef77-4582-9271-ffba5e4cabc9" diff --git a/arches_lingo/etl_modules/migrate_to_lingo.py b/arches_lingo/etl_modules/migrate_to_lingo.py index a1bdf164..37280ab5 100644 --- a/arches_lingo/etl_modules/migrate_to_lingo.py +++ b/arches_lingo/etl_modules/migrate_to_lingo.py @@ -2,18 +2,15 @@ import json import logging import uuid -from django.core.exceptions import ValidationError + from django.db import connection from django.db.models import OuterRef, Subquery +from django.db.models.functions import Coalesce from arches.app.datatypes.datatypes import DataTypeFactory from arches.app.etl_modules.save import save_to_tiles from arches.app.etl_modules.decorators import load_data_async -from arches.app.etl_modules.base_import_module import ( - BaseImportModule, - FileValidationError, -) +from arches.app.etl_modules.base_import_module import BaseImportModule from arches.app.models import models -from arches.app.models.concept import Concept from arches.app.models.models import LoadStaging, NodeGroup, LoadEvent from arches.app.models.system_settings import settings import arches_lingo.tasks as tasks @@ -66,12 +63,22 @@ def get_schemes(self, request): schemes = ( models.Concept.objects.filter(nodetype="ConceptScheme") .annotate( - prefLabel=Subquery( - models.Value.objects.filter( - valuetype_id="prefLabel", - concept_id=OuterRef("pk"), - language_id=settings.LANGUAGE_CODE, - ).values("value")[:1] + prefLabel=Coalesce( + Subquery( + models.Value.objects.filter( + valuetype_id="prefLabel", + concept_id=OuterRef("pk"), + language_id=settings.LANGUAGE_CODE, + ).values("value")[:1] + ), + Subquery( + models.Value.objects.filter( + valuetype_id="prefLabel", + concept_id=OuterRef("pk"), + ) + .order_by("language_id") + .values("value")[:1] + ), ) ) .order_by("prefLabel") @@ -82,7 +89,7 @@ def get_schemes(self, request): def etl_schemes(self, cursor, nodegroup_lookup, node_lookup, scheme_conceptid): schemes = [] for concept in models.Concept.objects.filter( - nodetype="ConceptScheme", pk=scheme_conceptid + pk=scheme_conceptid ).prefetch_related("value_set"): scheme_to_load = {"type": "Scheme", "tile_data": []} for value in concept.value_set.all(): @@ -432,7 +439,6 @@ def init_relationships( # Create Part of Scheme relationships - derived by recursively generating concept hierarchy & associating # concepts with their schemes part_of_scheme_tiles = [] - load_event = LoadEvent.objects.get(loadid=loadid) part_of_scheme_nodegroup = NodeGroup.objects.get( nodegroupid=CONCEPTS_PART_OF_SCHEME_NODEGROUP_ID ) @@ -464,7 +470,7 @@ def init_relationships( passes_validation=True, nodegroup_depth=0, source_description="Concept: Part of Scheme", - load_event=load_event, + load_event=LoadEvent(self.loadid), nodegroup=part_of_scheme_nodegroup, operation="insert", ) @@ -509,14 +515,13 @@ def write(self, request): self.userid, self.loadid, self.scheme_conceptid ) else: - response = self.run_load_task_async( - request, self.loadid, self.scheme_conceptid - ) + response = self.run_load_task_async(request, self.loadid) message = "Schemes and Concept Migration to Lingo Models Complete" return {"success": True, "data": message} def run_load_task(self, userid, loadid, scheme_conceptid): self.loadid = loadid # currently redundant, but be certain + self.scheme_conceptid = scheme_conceptid with connection.cursor() as cursor: @@ -573,11 +578,7 @@ def run_load_task(self, userid, loadid, scheme_conceptid): return {"success": False, "data": "failed"} @load_data_async - def run_load_task_async(self, request, scheme_conceptid): - self.userid = request.user.id - self.loadid = request.POST.get("loadid") - self.scheme_conceptid = request.POST.get("scheme") - + def run_load_task_async(self, request): migrate_rdm_to_lingo_task = tasks.migrate_rdm_to_lingo_task.apply_async( (self.userid, self.loadid, self.scheme_conceptid), ) diff --git a/arches_lingo/media/js/views/components/etl_modules/migrate-to-lingo.js b/arches_lingo/media/js/views/components/etl_modules/migrate-to-lingo.js index 9a5cac09..c8d53a1f 100644 --- a/arches_lingo/media/js/views/components/etl_modules/migrate-to-lingo.js +++ b/arches_lingo/media/js/views/components/etl_modules/migrate-to-lingo.js @@ -59,7 +59,6 @@ define([ params.activeTab("import"); self.formData.append('async', true); self.submit('write').then(data => { - console.log(data.results); }).fail(function(err) { console.log(err); self.alert( diff --git a/arches_lingo/pkg/graphs/resource_models/Scheme.json b/arches_lingo/pkg/graphs/resource_models/Scheme.json index 60b141bb..eac65967 100644 --- a/arches_lingo/pkg/graphs/resource_models/Scheme.json +++ b/arches_lingo/pkg/graphs/resource_models/Scheme.json @@ -2096,7 +2096,7 @@ "group_permissions": {}, "has_unpublished_changes": false, "iconclass": "", - "is_active": false, + "is_active": true, "is_copy_immutable": false, "isresource": true, "jsonldcontext": null, diff --git a/arches_lingo/tasks.py b/arches_lingo/tasks.py index 842c15e3..06544815 100644 --- a/arches_lingo/tasks.py +++ b/arches_lingo/tasks.py @@ -13,7 +13,7 @@ def migrate_rdm_to_lingo_task(userid, loadid, scheme_conceptid): try: Migrator = migrate_to_lingo.RDMMtoLingoMigrator(loadid=loadid) - Migrator.run_load_task(userid, loadid) + Migrator.run_load_task(userid, loadid, scheme_conceptid) load_event = models.LoadEvent.objects.get(loadid=loadid) status = _("Completed") if load_event.status == "indexed" else _("Failed") diff --git a/arches_lingo/templates/javascript.htm b/arches_lingo/templates/javascript.htm index 752cddd6..86576cff 100644 --- a/arches_lingo/templates/javascript.htm +++ b/arches_lingo/templates/javascript.htm @@ -7,7 +7,7 @@ {% block arches_translations %} {{ block.super }}
{% endblock arches_translations %} diff --git a/arches_lingo/templates/views/components/etl_modules/migrate-to-lingo.htm b/arches_lingo/templates/views/components/etl_modules/migrate-to-lingo.htm index 8c9cf1f4..0a5ca57d 100644 --- a/arches_lingo/templates/views/components/etl_modules/migrate-to-lingo.htm +++ b/arches_lingo/templates/views/components/etl_modules/migrate-to-lingo.htm @@ -13,9 +13,9 @@

RDM Migrator

{% blocktrans %}Click the start button to initiate the process for migrating concepts and schemes from the existing RDM to the Lingo resource models. You will need to have Celery running if your instance has more than 500 concepts.{% endblocktrans %}

-

+
-

+