Cherry pick backend fixes

arches_lingo/const.py

@@ -48,10 +48,6 @@
 SCHEME_NAME_TYPE_NODE = "ef87b588-11de-11ef-9493-0a58a9feac02"
 
 
-PREF_LABEL_VALUE_ID = "6ac8e471-476e-4fd0-b276-86e01a17bcc8"
-ALT_LABEL_VALUE_ID = "d9e362ca-c155-4569-9197-5583dd66f7e5"
-HIDDEN_LABEL_VALUE_ID = "18c46580-8c3c-48b7-9a6c-a0643708cb8b"
-
-# Old RDM concepts, values
-LANGUAGE_CONCEPT_ID = "a6b88323-7226-4428-8f41-3d5252e3a2a9"
-ENGLISH_VALUE_ID = "de978fd0-2819-4855-8858-8c089780f32c"
+### Lists and List Items ###
+LANGUAGES_LIST_ID = "55ce793b-a51a-4b25-811d-d08ea797f8c3"
+LABEL_LIST_ID = "deb847fc-f4c3-4e82-be19-04641579f129"

arches_lingo/utils/concept_builder.py

@@ -2,16 +2,11 @@
 
 from django.contrib.postgres.expressions import ArraySubquery
 from django.core.cache import caches
-from django.db.models import CharField, Exists, F, OuterRef, Value
+from django.db.models import CharField, F, OuterRef, Value
 from django.db.models.expressions import CombinedExpression
 from django.utils.translation import gettext_lazy as _
 
-from arches.app.models.models import (
-    Relation,
-    ResourceInstance,
-    TileModel,
-    Value as ConceptValue,
-)
+from arches.app.models.models import ResourceInstance, TileModel
 
 from arches_lingo.const import (
     SCHEMES_GRAPH_ID,
@@ -22,14 +17,10 @@
     CONCEPT_NAME_CONTENT_NODE,
     CONCEPT_NAME_LANGUAGE_NODE,
     CONCEPT_NAME_TYPE_NODE,
-    HIDDEN_LABEL_VALUE_ID,
-    LANGUAGE_CONCEPT_ID,
     SCHEME_NAME_NODEGROUP,
     SCHEME_NAME_CONTENT_NODE,
     SCHEME_NAME_LANGUAGE_NODE,
     SCHEME_NAME_TYPE_NODE,
-    PREF_LABEL_VALUE_ID,
-    ALT_LABEL_VALUE_ID,
 )
 from arches_lingo.utils.query_expressions import JsonbArrayElements
 
@@ -43,8 +34,6 @@ class ConceptBuilder:
     def __init__(self):
         self.schemes = ResourceInstance.objects.none()
 
-        # key=concept valueid (str) val=language code
-        self.language_concepts: dict[str:str] = {}
         # key=scheme resourceid (str) val=set of concept resourceids (str)
         self.top_concepts: dict[str : set[str]] = defaultdict(set)
         # key=concept resourceid (str) val=set of concept resourceids (str)
@@ -66,7 +55,6 @@ def __init__(self):
     def read_from_cache(self):
         from_cache = cache.get_many(
             [
-                "language_concepts",
                 "top_concepts",
                 "narrower_concepts",
                 "schemes",
@@ -76,7 +64,6 @@ def read_from_cache(self):
             ]
         )
         try:
-            self.language_concepts = from_cache["language_concepts"]
             self.top_concepts = from_cache["top_concepts"]
             self.narrower_concepts = from_cache["narrower_concepts"]
             self.schemes = from_cache["schemes"]
@@ -87,12 +74,10 @@ def read_from_cache(self):
             self.rebuild_cache()
 
     def rebuild_cache(self):
-        self.language_concepts_map()
         self.top_concepts_map()
         self.narrower_concepts_map()
         self.populate_schemes()
 
-        cache.set("language_concepts", self.language_concepts)
         cache.set("top_concepts", self.top_concepts)
         cache.set("narrower_concepts", self.narrower_concepts)
         cache.set("schemes", self.schemes)
@@ -101,16 +86,6 @@ def rebuild_cache(self):
         cache.set("broader_concepts", self.broader_concepts)
         cache.set("schemes_by_top_concept", self.schemes_by_top_concept)
 
-    @staticmethod
-    def human_label_type(value_id):
-        if value_id == PREF_LABEL_VALUE_ID:
-            return "prefLabel"
-        if value_id == ALT_LABEL_VALUE_ID:
-            return "altLabel"
-        if value_id == HIDDEN_LABEL_VALUE_ID:
-            return "hidden"
-        return "unknown"
-
     @staticmethod
     def resources_from_tiles(lookup_expression: str):
         return CombinedExpression(
@@ -137,20 +112,6 @@ def labels_subquery(label_nodegroup):
             ).values("data")
         )
 
-    def language_concepts_map(self):
-        language_preflabels = ConceptValue.objects.filter(
-            Exists(
-                Relation.objects.filter(
-                    conceptfrom=LANGUAGE_CONCEPT_ID,
-                    conceptto=OuterRef("concept_id"),
-                    relationtype="narrower",
-                )
-            ),
-            valuetype="prefLabel",
-        )
-        for language_label in language_preflabels:
-            self.language_concepts[str(language_label.pk)] = language_label.value
-
     def top_concepts_map(self):
         top_concept_of_tiles = (
             TileModel.objects.filter(nodegroup_id=TOP_CONCEPT_OF_NODE_AND_NODEGROUP)
@@ -198,15 +159,16 @@ def serialize_scheme(self, scheme: ResourceInstance, *, children=True):
         return data
 
     def serialize_scheme_label(self, label_tile: dict):
-        lang_code = self.language_concepts[label_tile[SCHEME_NAME_LANGUAGE_NODE][0]]
+        valuetype_id = label_tile[SCHEME_NAME_TYPE_NODE][0]["labels"][0]["value"]
+        language_id = label_tile[SCHEME_NAME_LANGUAGE_NODE][0]["labels"][0]["value"]
         localized_string_objs = label_tile[SCHEME_NAME_CONTENT_NODE].values()
         try:
             value = next(iter(localized_string_objs))["value"]
         except (StopIteration, KeyError):
             value = "Unknown"
         return {
-            "valuetype": self.human_label_type(label_tile[SCHEME_NAME_TYPE_NODE]),
-            "language": lang_code,
+            "valuetype_id": valuetype_id,
+            "language_id": language_id,
             "value": value,
         }
 
@@ -256,14 +218,15 @@ def add_broader_concept_recursive(self, working_parent_list, conceptid):
         )
 
     def serialize_concept_label(self, label_tile: dict):
-        lang_code = self.language_concepts[label_tile[CONCEPT_NAME_LANGUAGE_NODE][0]]
+        valuetype_id = label_tile[CONCEPT_NAME_TYPE_NODE][0]["labels"][0]["value"]
+        language_id = label_tile[CONCEPT_NAME_LANGUAGE_NODE][0]["labels"][0]["value"]
         localized_string_objs = label_tile[CONCEPT_NAME_CONTENT_NODE].values()
         try:
             value = next(iter(localized_string_objs))["value"]
         except (StopIteration, KeyError):
             value = "Unknown"
         return {
-            "valuetype": self.human_label_type(label_tile[CONCEPT_NAME_TYPE_NODE]),
-            "language": lang_code,
+            "valuetype_id": valuetype_id,
+            "language_id": language_id,
             "value": value,
         }

tests/tests.py

@@ -8,21 +8,17 @@
 # these tests can be run from the command line via
 # python manage.py test tests.tests --settings="tests.test_settings"
 
+from arches.app.datatypes.datatypes import DataTypeFactory
 from arches.app.models.models import (
-    Concept,
     GraphModel,
     Node,
     NodeGroup,
-    Relation,
     ResourceInstance,
     TileModel,
-    Value,
 )
 
 from arches_lingo.const import (
-    ENGLISH_VALUE_ID,
     CONCEPTS_GRAPH_ID,
-    LANGUAGE_CONCEPT_ID,
     SCHEMES_GRAPH_ID,
     TOP_CONCEPT_OF_NODE_AND_NODEGROUP,
     CLASSIFICATION_STATUS_NODEGROUP,
@@ -35,52 +31,73 @@
     SCHEME_NAME_CONTENT_NODE,
     SCHEME_NAME_LANGUAGE_NODE,
     SCHEME_NAME_TYPE_NODE,
-    PREF_LABEL_VALUE_ID,
+    LANGUAGES_LIST_ID,
+    LABEL_LIST_ID,
 )
 
 
-def setUpModule():
-    """Bootstrap just a few nodes as an alternative to loading the entire package."""
-    if not GraphModel.objects.filter(pk=SCHEMES_GRAPH_ID).exists():
+def localized_string(text, language="en", direction="ltr"):
+    return {language: {"value": text, "direction": direction}}
+
+
+class ViewTests(TestCase):
+    @classmethod
+    def mock_concept_and_scheme_graphs(cls):
+        """Bootstrap just a few nodes as an alternative to loading the entire package."""
+
         GraphModel.objects.create(pk=SCHEMES_GRAPH_ID, isresource=True)
         GraphModel.objects.create(pk=CONCEPTS_GRAPH_ID, isresource=True)
 
-        for nodegroup_id, node_id, node_name, datatype in [
+        for nodegroup_id, node_id, node_name, datatype, config in [
             (
                 TOP_CONCEPT_OF_NODE_AND_NODEGROUP,
                 TOP_CONCEPT_OF_NODE_AND_NODEGROUP,
                 "top_concept_of",
-                "concept-list",
+                "resource-instance-list",
+                {
+                    "graphs": [{"graphid": SCHEMES_GRAPH_ID, "name": "Scheme"}],
+                    "searchDsl": "",
+                    "searchString": "",
+                },
             ),
             (
                 CLASSIFICATION_STATUS_NODEGROUP,
                 CLASSIFICATION_STATUS_ASCRIBED_CLASSIFICATION_NODEID,
                 "classification_status_ascribed_classification",
-                "concept-list",
+                "resource-instance",
+                {
+                    "graphs": [{"graphid": CONCEPTS_GRAPH_ID, "name": "Concept"}],
+                    "searchDsl": "",
+                    "searchString": "",
+                },
             ),
             (
                 SCHEME_NAME_NODEGROUP,
                 SCHEME_NAME_CONTENT_NODE,
                 "appellative_status_ascribed_name_content",
-                "concept-list",
+                "string",
+                {"en": ""},
             ),
             (
                 SCHEME_NAME_NODEGROUP,
                 SCHEME_NAME_LANGUAGE_NODE,
                 "appellative_status_ascribed_name_language",
-                "string",
+                "reference",
+                {"controlledList": LANGUAGES_LIST_ID, "multiValue": True},
             ),
             (
                 CONCEPT_NAME_NODEGROUP,
                 CONCEPT_NAME_CONTENT_NODE,
                 "appellative_status_ascribed_name_content",
-                "concept-list",
+                "string",
+                {"en": ""},
             ),
             (
                 CONCEPT_NAME_NODEGROUP,
                 CONCEPT_NAME_LANGUAGE_NODE,
                 "appellative_status_ascribed_name_language",
-                "string",
+                "reference",
+                {"controlledList": LANGUAGES_LIST_ID, "multiValue": True},
             ),
         ]:
             NodeGroup.objects.update_or_create(
@@ -93,45 +110,34 @@ def setUpModule():
                 name=node_name,
                 istopnode=False,
                 datatype=datatype,
+                config=config,
                 isrequired=datatype == "string",
             )
 
-    Concept.objects.get_or_create(
-        conceptid=LANGUAGE_CONCEPT_ID,
-        nodetype_id="Concept",
-    )
-    Value.objects.get_or_create(
-        concept_id=LANGUAGE_CONCEPT_ID,
-        valueid=ENGLISH_VALUE_ID,
-        valuetype_id="prefLabel",
-        value="en",
-    )
-    Relation.objects.get_or_create(
-        conceptfrom_id=LANGUAGE_CONCEPT_ID,
-        conceptto_id=LANGUAGE_CONCEPT_ID,
-        relationtype_id="narrower",
-    )
-
-
-def localized_string(text, language="en", direction="ltr"):
-    return {language: {"value": text, "direction": direction}}
-
-
-class ViewTests(TestCase):
     @classmethod
     def setUpTestData(cls):
+        cls.mock_concept_and_scheme_graphs()
         cls.admin = User.objects.get(username="admin")
 
         # Create a scheme with five concepts, each one narrower than the last,
         # and each concept after the top concept also narrower than the top.
         cls.scheme = ResourceInstance.objects.create(graph_id=SCHEMES_GRAPH_ID)
+
+        reference = DataTypeFactory().get_instance("reference")
+        language_config = {"controlledList": LANGUAGES_LIST_ID}
+        label_config = {"controlledList": LABEL_LIST_ID}
+        prefLabel_reference_dt = reference.transform_value_for_tile(
+            "prefLabel", **label_config
+        )
+        en_reference_dt = reference.transform_value_for_tile("en", **language_config)
+
         TileModel.objects.create(
             resourceinstance=cls.scheme,
             nodegroup_id=SCHEME_NAME_NODEGROUP,
             data={
                 SCHEME_NAME_CONTENT_NODE: localized_string("Test Scheme"),
-                SCHEME_NAME_TYPE_NODE: PREF_LABEL_VALUE_ID,
-                SCHEME_NAME_LANGUAGE_NODE: [ENGLISH_VALUE_ID],
+                SCHEME_NAME_TYPE_NODE: prefLabel_reference_dt,
+                SCHEME_NAME_LANGUAGE_NODE: en_reference_dt,
             },
         )
 
@@ -150,8 +156,8 @@ def setUpTestData(cls):
                 nodegroup_id=CONCEPT_NAME_NODEGROUP,
                 data={
                     CONCEPT_NAME_CONTENT_NODE: localized_string(f"Concept {i + 1}"),
-                    CONCEPT_NAME_TYPE_NODE: PREF_LABEL_VALUE_ID,
-                    CONCEPT_NAME_LANGUAGE_NODE: [ENGLISH_VALUE_ID],
+                    CONCEPT_NAME_TYPE_NODE: prefLabel_reference_dt,
+                    CONCEPT_NAME_LANGUAGE_NODE: en_reference_dt,
                 },
             )
             # Create top concept/narrower tile
@@ -192,13 +198,12 @@ def setUpTestData(cls):
 
     def test_get_concept_trees(self):
         self.client.force_login(self.admin)
-        with self.assertNumQueries(6):
+        with self.assertNumQueries(5):
             # 1: session
             # 2: auth
-            # 3: select relations (to find languages)
-            # 4: select broader tiles, subquery for labels
-            # 5: select top concept tiles, subquery for labels
-            # 6: select schemes, subquery for labels
+            # 3: select broader tiles, subquery for labels
+            # 4: select top concept tiles, subquery for labels
+            # 5: select schemes, subquery for labels
             response = self.client.get(reverse("api-concepts"))
 
         self.assertEqual(response.status_code, 200)