Merge branch 'main' into 55_create_lingo_test_data

arches_lingo/const.py

@@ -4,7 +4,6 @@
 TOP_CONCEPT_OF_NODE_AND_NODEGROUP = "bf73e5b9-4888-11ee-8a8d-11afefc4bff7"
 
 # classification_status
-BROADER_NODE_AND_NODEGROUP = "f3f7bbea-0eb9-11ef-93db-0a58a9feac02"
 CLASSIFICATION_STATUS_NODEGROUP = "f3f7bbea-0eb9-11ef-93db-0a58a9feac02"
 CLASSIFICATION_STATUS_ASCRIBED_CLASSIFICATION_NODEID = (
     "0b531a82-0eba-11ef-93db-0a58a9feac02"
@@ -27,7 +26,7 @@
 # appellative_status
 CONCEPT_NAME_NODEGROUP = "ab9fee9c-0eb6-11ef-93db-0a58a9feac02"
 # appellative_status_ascribed_name_content
-CONCEPT_NAME_CONTENT_NODE = "bf73e695-4888-11ee-8a8d-11afefc4bff7"
+CONCEPT_NAME_CONTENT_NODE = "7357993a-0eb7-11ef-93db-0a58a9feac02"
 # appellative_status_ascribed_name_language
 CONCEPT_NAME_LANGUAGE_NODE = "a8ecaf54-0eb7-11ef-93db-0a58a9feac02"
 # appellative_status_ascribed_relation
@@ -49,8 +48,9 @@
 SCHEME_NAME_TYPE_NODE = "ef87b588-11de-11ef-9493-0a58a9feac02"
 
 
-PREF_LABEL_VALUE_ID = "3b8a03f1-9047-48e4-9ca0-b3fe887f6f9d"
-ALT_LABEL_VALUE_ID = "c02f97c5-da16-4ff0-864a-92c34da84e38"
+PREF_LABEL_VALUE_ID = "6ac8e471-476e-4fd0-b276-86e01a17bcc8"
+ALT_LABEL_VALUE_ID = "d9e362ca-c155-4569-9197-5583dd66f7e5"
+HIDDEN_LABEL_VALUE_ID = "18c46580-8c3c-48b7-9a6c-a0643708cb8b"
 
 # Old RDM concepts, values
 LANGUAGE_CONCEPT_ID = "845cc417-ef77-4582-9271-ffba5e4cabc9"

arches_lingo/etl_modules/migrate_to_lingo.py

@@ -2,18 +2,15 @@
 import json
 import logging
 import uuid
-from django.core.exceptions import ValidationError
+
 from django.db import connection
 from django.db.models import OuterRef, Subquery
+from django.db.models.functions import Coalesce
 from arches.app.datatypes.datatypes import DataTypeFactory
 from arches.app.etl_modules.save import save_to_tiles
 from arches.app.etl_modules.decorators import load_data_async
-from arches.app.etl_modules.base_import_module import (
-    BaseImportModule,
-    FileValidationError,
-)
+from arches.app.etl_modules.base_import_module import BaseImportModule
 from arches.app.models import models
-from arches.app.models.concept import Concept
 from arches.app.models.models import LoadStaging, NodeGroup, LoadEvent
 from arches.app.models.system_settings import settings
 import arches_lingo.tasks as tasks
@@ -66,12 +63,22 @@ def get_schemes(self, request):
         schemes = (
             models.Concept.objects.filter(nodetype="ConceptScheme")
             .annotate(
-                prefLabel=Subquery(
-                    models.Value.objects.filter(
-                        valuetype_id="prefLabel",
-                        concept_id=OuterRef("pk"),
-                        language_id=settings.LANGUAGE_CODE,
-                    ).values("value")[:1]
+                prefLabel=Coalesce(
+                    Subquery(
+                        models.Value.objects.filter(
+                            valuetype_id="prefLabel",
+                            concept_id=OuterRef("pk"),
+                            language_id=settings.LANGUAGE_CODE,
+                        ).values("value")[:1]
+                    ),
+                    Subquery(
+                        models.Value.objects.filter(
+                            valuetype_id="prefLabel",
+                            concept_id=OuterRef("pk"),
+                        )
+                        .order_by("language_id")
+                        .values("value")[:1]
+                    ),
                 )
             )
             .order_by("prefLabel")
@@ -82,7 +89,7 @@ def get_schemes(self, request):
     def etl_schemes(self, cursor, nodegroup_lookup, node_lookup, scheme_conceptid):
         schemes = []
         for concept in models.Concept.objects.filter(
-            nodetype="ConceptScheme", pk=scheme_conceptid
+            pk=scheme_conceptid
         ).prefetch_related("value_set"):
             scheme_to_load = {"type": "Scheme", "tile_data": []}
             for value in concept.value_set.all():
@@ -432,7 +439,6 @@ def init_relationships(
         # Create Part of Scheme relationships - derived by recursively generating concept hierarchy & associating
         # concepts with their schemes
         part_of_scheme_tiles = []
-        load_event = LoadEvent.objects.get(loadid=loadid)
         part_of_scheme_nodegroup = NodeGroup.objects.get(
             nodegroupid=CONCEPTS_PART_OF_SCHEME_NODEGROUP_ID
         )
@@ -464,7 +470,7 @@ def init_relationships(
                 passes_validation=True,
                 nodegroup_depth=0,
                 source_description="Concept: Part of Scheme",
-                load_event=load_event,
+                load_event=LoadEvent(self.loadid),
                 nodegroup=part_of_scheme_nodegroup,
                 operation="insert",
             )
@@ -509,14 +515,13 @@ def write(self, request):
                 self.userid, self.loadid, self.scheme_conceptid
             )
         else:
-            response = self.run_load_task_async(
-                request, self.loadid, self.scheme_conceptid
-            )
+            response = self.run_load_task_async(request, self.loadid)
         message = "Schemes and Concept Migration to Lingo Models Complete"
         return {"success": True, "data": message}
 
     def run_load_task(self, userid, loadid, scheme_conceptid):
         self.loadid = loadid  # currently redundant, but be certain
+        self.scheme_conceptid = scheme_conceptid
 
         with connection.cursor() as cursor:
 
@@ -573,11 +578,7 @@ def run_load_task(self, userid, loadid, scheme_conceptid):
                 return {"success": False, "data": "failed"}
 
     @load_data_async
-    def run_load_task_async(self, request, scheme_conceptid):
-        self.userid = request.user.id
-        self.loadid = request.POST.get("loadid")
-        self.scheme_conceptid = request.POST.get("scheme")
-
+    def run_load_task_async(self, request):
         migrate_rdm_to_lingo_task = tasks.migrate_rdm_to_lingo_task.apply_async(
             (self.userid, self.loadid, self.scheme_conceptid),
         )

arches_lingo/media/js/views/components/etl_modules/migrate-to-lingo.js

@@ -59,7 +59,6 @@ define([
                 params.activeTab("import");
                 self.formData.append('async', true);
                 self.submit('write').then(data => {
-                    console.log(data.results);
                 }).fail(function(err) {
                     console.log(err);
                     self.alert(

arches_lingo/pkg/graphs/resource_models/Scheme.json

@@ -2096,7 +2096,7 @@
             "group_permissions": {},
             "has_unpublished_changes": false,
             "iconclass": "",
-            "is_active": false,
+            "is_active": true,
             "is_copy_immutable": false,
             "isresource": true,
             "jsonldcontext": null,

arches_lingo/tasks.py

@@ -13,7 +13,7 @@ def migrate_rdm_to_lingo_task(userid, loadid, scheme_conceptid):
 
     try:
         Migrator = migrate_to_lingo.RDMMtoLingoMigrator(loadid=loadid)
-        Migrator.run_load_task(userid, loadid)
+        Migrator.run_load_task(userid, loadid, scheme_conceptid)
 
         load_event = models.LoadEvent.objects.get(loadid=loadid)
         status = _("Completed") if load_event.status == "indexed" else _("Failed")

arches_lingo/templates/javascript.htm

@@ -7,7 +7,7 @@
 {% block arches_translations %}
 {{ block.super }}
 <div class='arches-translations'
-    selectAThesaurusToMigrate='{% trans "Select a Thesaurus to Migrate to the Lingo Models" as required %} "{{ selectAThesaurusToMigrate|escapejs }}"'
-    schemesMigrated='{% trans "Migrated Scheme" as required %} "{{ schemesMigrated|escapejs }}"'
+    select-a-thesaurus-to-migrate='{% trans "Select a Thesaurus to Migrate to the Lingo Models" as selectAThesaurusToMigrate %} "{{ selectAThesaurusToMigrate|escapejs }}"'
+    schemes-migrated='{% trans "Migrated Scheme" as schemesMigrated %} "{{ schemesMigrated|escapejs }}"'
 ></div>
 {% endblock arches_translations %}

arches_lingo/templates/views/components/etl_modules/migrate-to-lingo.htm

@@ -13,9 +13,9 @@ <h2>RDM Migrator</h2>
             {% blocktrans %}Click the start button to initiate the process for migrating concepts and schemes from the existing RDM to the Lingo resource models. You will need to have Celery running if your instance has more than 500 concepts.{% endblocktrans %}
         </p>
         <div class="etl-module-component-block">
-            <h4>
+            <div class="h4">
                 <label for="resource-model-select" data-bind="text: $root.translations.selectAThesaurusToMigrate"></label>
-            </h4>
+            </div>
             <select id="resource-model-select" data-bind="
                 value: selectedScheme,
                 options: schemes,

arches_lingo/views.py

@@ -22,11 +22,13 @@
 from arches_lingo.const import (
     SCHEMES_GRAPH_ID,
     TOP_CONCEPT_OF_NODE_AND_NODEGROUP,
-    BROADER_NODE_AND_NODEGROUP,
+    CLASSIFICATION_STATUS_NODEGROUP,
+    CLASSIFICATION_STATUS_ASCRIBED_CLASSIFICATION_NODEID,
     CONCEPT_NAME_NODEGROUP,
     CONCEPT_NAME_CONTENT_NODE,
     CONCEPT_NAME_LANGUAGE_NODE,
     CONCEPT_NAME_TYPE_NODE,
+    HIDDEN_LABEL_VALUE_ID,
     SCHEME_NAME_NODEGROUP,
     SCHEME_NAME_CONTENT_NODE,
     SCHEME_NAME_LANGUAGE_NODE,
@@ -36,7 +38,7 @@
 )
 
 TOP_CONCEPT_OF_LOOKUP = f"data__{TOP_CONCEPT_OF_NODE_AND_NODEGROUP}"
-BROADER_LOOKUP = f"data__{BROADER_NODE_AND_NODEGROUP}"
+BROADER_LOOKUP = f"data__{CLASSIFICATION_STATUS_ASCRIBED_CLASSIFICATION_NODEID}"
 
 
 class JsonbArrayElements(Func):
@@ -69,6 +71,8 @@ def human_label_type(value_id):
             return "prefLabel"
         if value_id == ALT_LABEL_VALUE_ID:
             return "altLabel"
+        if value_id == HIDDEN_LABEL_VALUE_ID:
+            return "hidden"
         return "unknown"
 
     @staticmethod
@@ -127,7 +131,7 @@ def top_concepts_map(self):
 
     def narrower_concepts_map(self):
         broader_concept_tiles = (
-            TileModel.objects.filter(nodegroup_id=BROADER_NODE_AND_NODEGROUP)
+            TileModel.objects.filter(nodegroup_id=CLASSIFICATION_STATUS_NODEGROUP)
             .annotate(broader_concept=self.resources_from_tiles(BROADER_LOOKUP))
             .annotate(labels=self.labels_subquery(CONCEPT_NAME_NODEGROUP))
             .values("resourceinstance_id", "broader_concept", "labels")
@@ -150,12 +154,13 @@ def serialize_scheme(self, scheme: ResourceInstance):
 
     def serialize_scheme_label(self, label_tile: dict):
         lang_code = self.language_concepts[label_tile[SCHEME_NAME_LANGUAGE_NODE][0]]
+        localized_string_objs = label_tile[SCHEME_NAME_CONTENT_NODE].values()
         try:
-            value = label_tile[SCHEME_NAME_CONTENT_NODE][lang_code]["value"]
-        except KeyError:
+            value = next(iter(localized_string_objs))["value"]
+        except (StopIteration, KeyError):
             value = "Unknown"
         return {
-            "valuetype": self.human_label_type(label_tile[SCHEME_NAME_TYPE_NODE][0]),
+            "valuetype": self.human_label_type(label_tile[SCHEME_NAME_TYPE_NODE]),
             "language": lang_code,
             "value": value,
         }
@@ -174,12 +179,13 @@ def serialize_concept(self, conceptid: str):
 
     def serialize_concept_label(self, label_tile: dict):
         lang_code = self.language_concepts[label_tile[CONCEPT_NAME_LANGUAGE_NODE][0]]
+        localized_string_objs = label_tile[CONCEPT_NAME_CONTENT_NODE].values()
         try:
-            value = label_tile[CONCEPT_NAME_CONTENT_NODE][lang_code]["value"]
-        except KeyError:
+            value = next(iter(localized_string_objs))["value"]
+        except (StopIteration, KeyError):
             value = "Unknown"
         return {
-            "valuetype": self.human_label_type(label_tile[CONCEPT_NAME_TYPE_NODE][0]),
+            "valuetype": self.human_label_type(label_tile[CONCEPT_NAME_TYPE_NODE]),
             "language": lang_code,
             "value": value,
         }