From 76fa5883f65b8ea96f6a04c6f9f378024210734e Mon Sep 17 00:00:00 2001 From: Johnathan Clementi Date: Thu, 24 Oct 2024 10:01:20 -0400 Subject: [PATCH 1/6] Update serialization of schemes and concepts for refdt #107 --- arches_lingo/utils/concept_builder.py | 48 +++++---------------------- 1 file changed, 8 insertions(+), 40 deletions(-) diff --git a/arches_lingo/utils/concept_builder.py b/arches_lingo/utils/concept_builder.py index 71ba8fe7..d2e63acf 100644 --- a/arches_lingo/utils/concept_builder.py +++ b/arches_lingo/utils/concept_builder.py @@ -22,14 +22,10 @@ CONCEPT_NAME_CONTENT_NODE, CONCEPT_NAME_LANGUAGE_NODE, CONCEPT_NAME_TYPE_NODE, - HIDDEN_LABEL_VALUE_ID, - LANGUAGE_CONCEPT_ID, SCHEME_NAME_NODEGROUP, SCHEME_NAME_CONTENT_NODE, SCHEME_NAME_LANGUAGE_NODE, SCHEME_NAME_TYPE_NODE, - PREF_LABEL_VALUE_ID, - ALT_LABEL_VALUE_ID, ) from arches_lingo.utils.query_expressions import JsonbArrayElements @@ -43,8 +39,6 @@ class ConceptBuilder: def __init__(self): self.schemes = ResourceInstance.objects.none() - # key=concept valueid (str) val=language code - self.language_concepts: dict[str:str] = {} # key=scheme resourceid (str) val=set of concept resourceids (str) self.top_concepts: dict[str : set[str]] = defaultdict(set) # key=concept resourceid (str) val=set of concept resourceids (str) @@ -66,7 +60,6 @@ def __init__(self): def read_from_cache(self): from_cache = cache.get_many( [ - "language_concepts", "top_concepts", "narrower_concepts", "schemes", @@ -76,7 +69,6 @@ def read_from_cache(self): ] ) try: - self.language_concepts = from_cache["language_concepts"] self.top_concepts = from_cache["top_concepts"] self.narrower_concepts = from_cache["narrower_concepts"] self.schemes = from_cache["schemes"] @@ -87,12 +79,10 @@ def read_from_cache(self): self.rebuild_cache() def rebuild_cache(self): - self.language_concepts_map() self.top_concepts_map() self.narrower_concepts_map() self.populate_schemes() - cache.set("language_concepts", self.language_concepts) cache.set("top_concepts", self.top_concepts) cache.set("narrower_concepts", self.narrower_concepts) cache.set("schemes", self.schemes) @@ -101,16 +91,6 @@ def rebuild_cache(self): cache.set("broader_concepts", self.broader_concepts) cache.set("schemes_by_top_concept", self.schemes_by_top_concept) - @staticmethod - def human_label_type(value_id): - if value_id == PREF_LABEL_VALUE_ID: - return "prefLabel" - if value_id == ALT_LABEL_VALUE_ID: - return "altLabel" - if value_id == HIDDEN_LABEL_VALUE_ID: - return "hidden" - return "unknown" - @staticmethod def resources_from_tiles(lookup_expression: str): return CombinedExpression( @@ -137,20 +117,6 @@ def labels_subquery(label_nodegroup): ).values("data") ) - def language_concepts_map(self): - language_preflabels = ConceptValue.objects.filter( - Exists( - Relation.objects.filter( - conceptfrom=LANGUAGE_CONCEPT_ID, - conceptto=OuterRef("concept_id"), - relationtype="narrower", - ) - ), - valuetype="prefLabel", - ) - for language_label in language_preflabels: - self.language_concepts[str(language_label.pk)] = language_label.value - def top_concepts_map(self): top_concept_of_tiles = ( TileModel.objects.filter(nodegroup_id=TOP_CONCEPT_OF_NODE_AND_NODEGROUP) @@ -198,15 +164,16 @@ def serialize_scheme(self, scheme: ResourceInstance, *, children=True): return data def serialize_scheme_label(self, label_tile: dict): - lang_code = self.language_concepts[label_tile[SCHEME_NAME_LANGUAGE_NODE][0]] + valuetype_id = label_tile[SCHEME_NAME_TYPE_NODE][0]["labels"][0]["value"] + language_id = label_tile[SCHEME_NAME_LANGUAGE_NODE][0]["labels"][0]["value"] localized_string_objs = label_tile[SCHEME_NAME_CONTENT_NODE].values() try: value = next(iter(localized_string_objs))["value"] except (StopIteration, KeyError): value = "Unknown" return { - "valuetype": self.human_label_type(label_tile[SCHEME_NAME_TYPE_NODE]), - "language": lang_code, + "valuetype_id": valuetype_id, + "language_id": language_id, "value": value, } @@ -256,14 +223,15 @@ def add_broader_concept_recursive(self, working_parent_list, conceptid): ) def serialize_concept_label(self, label_tile: dict): - lang_code = self.language_concepts[label_tile[CONCEPT_NAME_LANGUAGE_NODE][0]] + valuetype_id = label_tile[CONCEPT_NAME_TYPE_NODE][0]["labels"][0]["value"] + language_id = label_tile[CONCEPT_NAME_LANGUAGE_NODE][0]["labels"][0]["value"] localized_string_objs = label_tile[CONCEPT_NAME_CONTENT_NODE].values() try: value = next(iter(localized_string_objs))["value"] except (StopIteration, KeyError): value = "Unknown" return { - "valuetype": self.human_label_type(label_tile[CONCEPT_NAME_TYPE_NODE]), - "language": lang_code, + "valuetype_id": valuetype_id, + "language_id": language_id, "value": value, } From 4a3f915c066dabdedc861d6cd97ed3096ed7c360 Mon Sep 17 00:00:00 2001 From: Johnathan Clementi Date: Thu, 24 Oct 2024 10:03:06 -0400 Subject: [PATCH 2/6] Update tests to reflect usage of refdt #107 --- arches_lingo/const.py | 10 ++--- tests/tests.py | 95 +++++++++++++++++++++++-------------------- 2 files changed, 54 insertions(+), 51 deletions(-) diff --git a/arches_lingo/const.py b/arches_lingo/const.py index caa01568..cd3dcee5 100644 --- a/arches_lingo/const.py +++ b/arches_lingo/const.py @@ -48,10 +48,6 @@ SCHEME_NAME_TYPE_NODE = "ef87b588-11de-11ef-9493-0a58a9feac02" -PREF_LABEL_VALUE_ID = "6ac8e471-476e-4fd0-b276-86e01a17bcc8" -ALT_LABEL_VALUE_ID = "d9e362ca-c155-4569-9197-5583dd66f7e5" -HIDDEN_LABEL_VALUE_ID = "18c46580-8c3c-48b7-9a6c-a0643708cb8b" - -# Old RDM concepts, values -LANGUAGE_CONCEPT_ID = "a6b88323-7226-4428-8f41-3d5252e3a2a9" -ENGLISH_VALUE_ID = "de978fd0-2819-4855-8858-8c089780f32c" +### Lists and List Items ### +LANGUAGES_LIST_ID = "55ce793b-a51a-4b25-811d-d08ea797f8c3" +LABEL_LIST_ID = "deb847fc-f4c3-4e82-be19-04641579f129" diff --git a/tests/tests.py b/tests/tests.py index b2afca78..87164e9a 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -8,6 +8,7 @@ # these tests can be run from the command line via # python manage.py test tests.tests --settings="tests.test_settings" +from arches.app.datatypes.datatypes import DataTypeFactory from arches.app.models.models import ( Concept, GraphModel, @@ -20,9 +21,7 @@ ) from arches_lingo.const import ( - ENGLISH_VALUE_ID, CONCEPTS_GRAPH_ID, - LANGUAGE_CONCEPT_ID, SCHEMES_GRAPH_ID, TOP_CONCEPT_OF_NODE_AND_NODEGROUP, CLASSIFICATION_STATUS_NODEGROUP, @@ -35,52 +34,72 @@ SCHEME_NAME_CONTENT_NODE, SCHEME_NAME_LANGUAGE_NODE, SCHEME_NAME_TYPE_NODE, - PREF_LABEL_VALUE_ID, + LANGUAGES_LIST_ID, + LABEL_LIST_ID, ) -def setUpModule(): - """Bootstrap just a few nodes as an alternative to loading the entire package.""" - if not GraphModel.objects.filter(pk=SCHEMES_GRAPH_ID).exists(): +def localized_string(text, language="en", direction="ltr"): + return {language: {"value": text, "direction": direction}} + + +class ViewTests(TestCase): + def setUpModule(): + """Bootstrap just a few nodes as an alternative to loading the entire package.""" + GraphModel.objects.create(pk=SCHEMES_GRAPH_ID, isresource=True) GraphModel.objects.create(pk=CONCEPTS_GRAPH_ID, isresource=True) - for nodegroup_id, node_id, node_name, datatype in [ + for nodegroup_id, node_id, node_name, datatype, config in [ ( TOP_CONCEPT_OF_NODE_AND_NODEGROUP, TOP_CONCEPT_OF_NODE_AND_NODEGROUP, "top_concept_of", - "concept-list", + "resource-instance-list", + { + "graphs": [{"graphid": SCHEMES_GRAPH_ID, "name": "Scheme"}], + "searchDsl": "", + "searchString": "", + }, ), ( CLASSIFICATION_STATUS_NODEGROUP, CLASSIFICATION_STATUS_ASCRIBED_CLASSIFICATION_NODEID, "classification_status_ascribed_classification", - "concept-list", + "resource-instance", + { + "graphs": [{"graphid": CONCEPTS_GRAPH_ID, "name": "Concept"}], + "searchDsl": "", + "searchString": "", + }, ), ( SCHEME_NAME_NODEGROUP, SCHEME_NAME_CONTENT_NODE, "appellative_status_ascribed_name_content", - "concept-list", + "string", + {"en": ""}, ), ( SCHEME_NAME_NODEGROUP, SCHEME_NAME_LANGUAGE_NODE, "appellative_status_ascribed_name_language", - "string", + "reference", + {"controlledList": LANGUAGES_LIST_ID, "multiValue": True}, ), ( CONCEPT_NAME_NODEGROUP, CONCEPT_NAME_CONTENT_NODE, "appellative_status_ascribed_name_content", - "concept-list", + "string", + {"en": ""}, ), ( CONCEPT_NAME_NODEGROUP, CONCEPT_NAME_LANGUAGE_NODE, "appellative_status_ascribed_name_language", - "string", + "reference", + {"controlledList": LANGUAGES_LIST_ID, "multiValue": True}, ), ]: NodeGroup.objects.update_or_create( @@ -93,45 +112,34 @@ def setUpModule(): name=node_name, istopnode=False, datatype=datatype, + config=config, isrequired=datatype == "string", ) - Concept.objects.get_or_create( - conceptid=LANGUAGE_CONCEPT_ID, - nodetype_id="Concept", - ) - Value.objects.get_or_create( - concept_id=LANGUAGE_CONCEPT_ID, - valueid=ENGLISH_VALUE_ID, - valuetype_id="prefLabel", - value="en", - ) - Relation.objects.get_or_create( - conceptfrom_id=LANGUAGE_CONCEPT_ID, - conceptto_id=LANGUAGE_CONCEPT_ID, - relationtype_id="narrower", - ) - - -def localized_string(text, language="en", direction="ltr"): - return {language: {"value": text, "direction": direction}} - - -class ViewTests(TestCase): @classmethod def setUpTestData(cls): + cls.setUpModule() cls.admin = User.objects.get(username="admin") # Create a scheme with five concepts, each one narrower than the last, # and each concept after the top concept also narrower than the top. cls.scheme = ResourceInstance.objects.create(graph_id=SCHEMES_GRAPH_ID) + + reference = DataTypeFactory().get_instance("reference") + language_config = {"controlledList": LANGUAGES_LIST_ID} + label_config = {"controlledList": LABEL_LIST_ID} + prefLabel_reference_dt = reference.transform_value_for_tile( + "prefLabel", **label_config + ) + en_reference_dt = reference.transform_value_for_tile("en", **language_config) + TileModel.objects.create( resourceinstance=cls.scheme, nodegroup_id=SCHEME_NAME_NODEGROUP, data={ SCHEME_NAME_CONTENT_NODE: localized_string("Test Scheme"), - SCHEME_NAME_TYPE_NODE: PREF_LABEL_VALUE_ID, - SCHEME_NAME_LANGUAGE_NODE: [ENGLISH_VALUE_ID], + SCHEME_NAME_TYPE_NODE: prefLabel_reference_dt, + SCHEME_NAME_LANGUAGE_NODE: en_reference_dt, }, ) @@ -150,8 +158,8 @@ def setUpTestData(cls): nodegroup_id=CONCEPT_NAME_NODEGROUP, data={ CONCEPT_NAME_CONTENT_NODE: localized_string(f"Concept {i + 1}"), - CONCEPT_NAME_TYPE_NODE: PREF_LABEL_VALUE_ID, - CONCEPT_NAME_LANGUAGE_NODE: [ENGLISH_VALUE_ID], + CONCEPT_NAME_TYPE_NODE: prefLabel_reference_dt, + CONCEPT_NAME_LANGUAGE_NODE: en_reference_dt, }, ) # Create top concept/narrower tile @@ -192,13 +200,12 @@ def setUpTestData(cls): def test_get_concept_trees(self): self.client.force_login(self.admin) - with self.assertNumQueries(6): + with self.assertNumQueries(5): # 1: session # 2: auth - # 3: select relations (to find languages) - # 4: select broader tiles, subquery for labels - # 5: select top concept tiles, subquery for labels - # 6: select schemes, subquery for labels + # 3: select broader tiles, subquery for labels + # 4: select top concept tiles, subquery for labels + # 5: select schemes, subquery for labels response = self.client.get(reverse("api-concepts")) self.assertEqual(response.status_code, 200) From 2265502e3c0893eb4cf7d9d66a1e7a582eae3abc Mon Sep 17 00:00:00 2001 From: Johnathan Clementi Date: Thu, 24 Oct 2024 10:40:05 -0400 Subject: [PATCH 3/6] Don't assume lingo lists have been loaded #107 --- tests/tests.py | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 87164e9a..aa55fe91 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -8,7 +8,6 @@ # these tests can be run from the command line via # python manage.py test tests.tests --settings="tests.test_settings" -from arches.app.datatypes.datatypes import DataTypeFactory from arches.app.models.models import ( Concept, GraphModel, @@ -35,7 +34,6 @@ SCHEME_NAME_LANGUAGE_NODE, SCHEME_NAME_TYPE_NODE, LANGUAGES_LIST_ID, - LABEL_LIST_ID, ) @@ -125,14 +123,35 @@ def setUpTestData(cls): # and each concept after the top concept also narrower than the top. cls.scheme = ResourceInstance.objects.create(graph_id=SCHEMES_GRAPH_ID) - reference = DataTypeFactory().get_instance("reference") - language_config = {"controlledList": LANGUAGES_LIST_ID} - label_config = {"controlledList": LABEL_LIST_ID} - prefLabel_reference_dt = reference.transform_value_for_tile( - "prefLabel", **label_config - ) - en_reference_dt = reference.transform_value_for_tile("en", **language_config) - + prefLabel_reference_dt = [ + { + "uri": "https://rdm.dev.fargeo.com/plugins/controlled-list-manager/item/2e3a2045-b44e-47fc-a27b-3078b17e08e4", + "labels": [ + { + "id": "6ac8e471-476e-4fd0-b276-86e01a17bcc8", + "value": "prefLabel", + "language_id": "en", + "list_item_id": "2e3a2045-b44e-47fc-a27b-3078b17e08e4", + "valuetype_id": "prefLabel", + } + ], + "listid": "deb847fc-f4c3-4e82-be19-04641579f129", + } + ] + en_reference_dt = [ + { + "uri": "https://rdm.dev.fargeo.com/plugins/controlled-list-manager/item/845cc417-ef77-4582-9271-ffba5e4cabc9", + "labels": [ + { + "id": "de978fd0-2819-4855-8858-8c089780f32c", + "value": "en", + "language_id": "en", + "list_item_id": "845cc417-ef77-4582-9271-ffba5e4cabc9", + "valuetype_id": "prefLabel", + } + ], + } + ] TileModel.objects.create( resourceinstance=cls.scheme, nodegroup_id=SCHEME_NAME_NODEGROUP, From ab77fe350dfef46b279477ebab191d318fffe38d Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Thu, 24 Oct 2024 11:04:00 -0400 Subject: [PATCH 4/6] Rename helper method, remove imports --- tests/tests.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index aa55fe91..708c6d04 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -9,14 +9,11 @@ # python manage.py test tests.tests --settings="tests.test_settings" from arches.app.models.models import ( - Concept, GraphModel, Node, NodeGroup, - Relation, ResourceInstance, TileModel, - Value, ) from arches_lingo.const import ( @@ -42,7 +39,8 @@ def localized_string(text, language="en", direction="ltr"): class ViewTests(TestCase): - def setUpModule(): + @classmethod + def mock_concept_and_scheme_graphs(cls): """Bootstrap just a few nodes as an alternative to loading the entire package.""" GraphModel.objects.create(pk=SCHEMES_GRAPH_ID, isresource=True) @@ -116,7 +114,7 @@ def setUpModule(): @classmethod def setUpTestData(cls): - cls.setUpModule() + cls.mock_concept_and_scheme_graphs() cls.admin = User.objects.get(username="admin") # Create a scheme with five concepts, each one narrower than the last, From 6b4f5a632aa08ea5796366a5beec2a8247d91e4f Mon Sep 17 00:00:00 2001 From: Jacob Walls Date: Tue, 29 Oct 2024 14:22:10 -0400 Subject: [PATCH 5/6] Remove additional unused imports --- arches_lingo/utils/concept_builder.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/arches_lingo/utils/concept_builder.py b/arches_lingo/utils/concept_builder.py index d2e63acf..cd702be7 100644 --- a/arches_lingo/utils/concept_builder.py +++ b/arches_lingo/utils/concept_builder.py @@ -2,16 +2,11 @@ from django.contrib.postgres.expressions import ArraySubquery from django.core.cache import caches -from django.db.models import CharField, Exists, F, OuterRef, Value +from django.db.models import CharField, F, OuterRef, Value from django.db.models.expressions import CombinedExpression from django.utils.translation import gettext_lazy as _ -from arches.app.models.models import ( - Relation, - ResourceInstance, - TileModel, - Value as ConceptValue, -) +from arches.app.models.models import ResourceInstance, TileModel from arches_lingo.const import ( SCHEMES_GRAPH_ID, From 974709ca16947210da0fac54e08e21a335c8ed3f Mon Sep 17 00:00:00 2001 From: Johnathan Clementi Date: Tue, 29 Oct 2024 14:44:50 -0400 Subject: [PATCH 6/6] Revert "Don't assume lingo lists have been loaded #107" --- tests/tests.py | 39 ++++++++++----------------------------- 1 file changed, 10 insertions(+), 29 deletions(-) diff --git a/tests/tests.py b/tests/tests.py index 708c6d04..987e5077 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -8,6 +8,7 @@ # these tests can be run from the command line via # python manage.py test tests.tests --settings="tests.test_settings" +from arches.app.datatypes.datatypes import DataTypeFactory from arches.app.models.models import ( GraphModel, Node, @@ -31,6 +32,7 @@ SCHEME_NAME_LANGUAGE_NODE, SCHEME_NAME_TYPE_NODE, LANGUAGES_LIST_ID, + LABEL_LIST_ID, ) @@ -121,35 +123,14 @@ def setUpTestData(cls): # and each concept after the top concept also narrower than the top. cls.scheme = ResourceInstance.objects.create(graph_id=SCHEMES_GRAPH_ID) - prefLabel_reference_dt = [ - { - "uri": "https://rdm.dev.fargeo.com/plugins/controlled-list-manager/item/2e3a2045-b44e-47fc-a27b-3078b17e08e4", - "labels": [ - { - "id": "6ac8e471-476e-4fd0-b276-86e01a17bcc8", - "value": "prefLabel", - "language_id": "en", - "list_item_id": "2e3a2045-b44e-47fc-a27b-3078b17e08e4", - "valuetype_id": "prefLabel", - } - ], - "listid": "deb847fc-f4c3-4e82-be19-04641579f129", - } - ] - en_reference_dt = [ - { - "uri": "https://rdm.dev.fargeo.com/plugins/controlled-list-manager/item/845cc417-ef77-4582-9271-ffba5e4cabc9", - "labels": [ - { - "id": "de978fd0-2819-4855-8858-8c089780f32c", - "value": "en", - "language_id": "en", - "list_item_id": "845cc417-ef77-4582-9271-ffba5e4cabc9", - "valuetype_id": "prefLabel", - } - ], - } - ] + reference = DataTypeFactory().get_instance("reference") + language_config = {"controlledList": LANGUAGES_LIST_ID} + label_config = {"controlledList": LABEL_LIST_ID} + prefLabel_reference_dt = reference.transform_value_for_tile( + "prefLabel", **label_config + ) + en_reference_dt = reference.transform_value_for_tile("en", **language_config) + TileModel.objects.create( resourceinstance=cls.scheme, nodegroup_id=SCHEME_NAME_NODEGROUP,