diff --git a/arches_lingo/views/trees.py b/arches_lingo/concepts.py similarity index 73% rename from arches_lingo/views/trees.py rename to arches_lingo/concepts.py index 858a6b33..1c2c514a 100644 --- a/arches_lingo/views/trees.py +++ b/arches_lingo/concepts.py @@ -2,19 +2,9 @@ from django.contrib.postgres.expressions import ArraySubquery from django.core.cache import caches -from django.core.paginator import Paginator -from django.db.models import ( - CharField, - Exists, - FloatField, - F, - OuterRef, - Value, -) -from django.db.models.expressions import CombinedExpression, Func +from django.db.models import CharField, Exists, F, OuterRef, Value +from django.db.models.expressions import CombinedExpression from django.utils.translation import gettext_lazy as _ -from django.utils.decorators import method_decorator -from django.views.generic import View from arches.app.models.models import ( Relation, @@ -22,9 +12,6 @@ TileModel, Value as ConceptValue, ) -from arches.app.models.system_settings import settings -from arches.app.utils.decorators import group_required -from arches.app.utils.response import JSONResponse from arches_lingo.const import ( SCHEMES_GRAPH_ID, @@ -44,7 +31,7 @@ PREF_LABEL_VALUE_ID, ALT_LABEL_VALUE_ID, ) -from arches_lingo.models import VwLabelValue +from arches_lingo.query_utils import JsonbArrayElements TOP_CONCEPT_OF_LOOKUP = f"data__{TOP_CONCEPT_OF_NODE_AND_NODEGROUP}" BROADER_LOOKUP = f"data__{CLASSIFICATION_STATUS_ASCRIBED_CLASSIFICATION_NODEID}" @@ -52,25 +39,8 @@ cache = caches["lingo"] -class JsonbArrayElements(Func): - """https://forum.djangoproject.com/t/django-4-2-behavior-change-when-using-arrayagg-on-unnested-arrayfield-postgresql-specific/21547/5""" - - arity = 1 - contains_subquery = True - function = "JSONB_ARRAY_ELEMENTS" - - -class LevenshteinLessEqual(Func): - arity = 3 - function = "LEVENSHTEIN_LESS_EQUAL" - - -@method_decorator( - group_required("RDM Administrator", raise_exception=True), name="dispatch" -) -class ConceptTreeView(View): +class ConceptBuilder: def __init__(self): - super().__init__() self.schemes = ResourceInstance.objects.none() # key=concept valueid (str) val=language code @@ -280,11 +250,11 @@ def add_broader_concept_recursive(self, working_parent_list, conceptid): schemes = sorted(self.schemes_by_top_concept[conceptid]) working_parent_list.insert(0, schemes[0]) return working_parent_list - else: - working_parent_list.insert(0, first_broader_conceptid) - return self.add_broader_concept_recursive( - working_parent_list, first_broader_conceptid - ) + + working_parent_list.insert(0, first_broader_conceptid) + return self.add_broader_concept_recursive( + working_parent_list, first_broader_conceptid + ) def serialize_concept_label(self, label_tile: dict): lang_code = self.language_concepts[label_tile[CONCEPT_NAME_LANGUAGE_NODE][0]] @@ -298,74 +268,3 @@ def serialize_concept_label(self, label_tile: dict): "language": lang_code, "value": value, } - - def get(self, request): - data = { - "schemes": [self.serialize_scheme(scheme) for scheme in self.schemes], - } - # Todo: filter by nodegroup permissions - return JSONResponse(data) - - -@method_decorator( - group_required("RDM Administrator", raise_exception=True), name="dispatch" -) -class ValueSearchView(ConceptTreeView): - def get(self, request): - term = request.GET.get("term") - max_edit_distance = request.GET.get( - "maxEditDistance", self.default_sensitivity() - ) - page_number = request.GET.get("page", 1) - items_per_page = request.GET.get("items", 25) - - concept_query = VwLabelValue.objects.all() - if term: - concept_query = ( - concept_query.annotate( - edit_distance=LevenshteinLessEqual( - F("value"), - Value(term), - Value(max_edit_distance), - output_field=FloatField(), - ) - ) - .filter(edit_distance__lte=max_edit_distance) - .order_by("edit_distance") - ) - else: - concept_query = concept_query.order_by("concept_id") - concept_query = concept_query.values_list("concept_id", flat=True).distinct() - - paginator = Paginator(concept_query, items_per_page) - page = paginator.get_page(page_number) - - data = [ - self.serialize_concept(str(concept_uuid), parents=True, children=False) - for concept_uuid in page - ] - - # Todo: filter by nodegroup permissions - return JSONResponse( - { - "current_page": page.number, - "total_pages": paginator.num_pages, - "results_per_page": paginator.per_page, - "total_results": paginator.count, - "data": data, - } - ) - - @staticmethod - def default_sensitivity(): - """Remains to be seen whether the existing elastic sensitivity setting - should be the fallback, but stub something out for now. - This sensitivity setting is actually inversely related to edit distance, - because it's prefix_length in elastic, not fuzziness, so invert it. - """ - elastic_prefix_length = settings.SEARCH_TERM_SENSITIVITY - if elastic_prefix_length <= 0: - return 5 - if elastic_prefix_length >= 5: - return 0 - return int(5 - elastic_prefix_length) diff --git a/arches_lingo/migrations/0001_initial.py b/arches_lingo/migrations/0001_initial.py index 99ebdceb..52dc9f24 100644 --- a/arches_lingo/migrations/0001_initial.py +++ b/arches_lingo/migrations/0001_initial.py @@ -9,7 +9,7 @@ class FuzzyStrMatchExtension(CreateExtension): def __init__(self): - self.name = "fuzzystrmatch" + super().__init__("fuzzystrmatch") class Migration(migrations.Migration): diff --git a/arches_lingo/models.py b/arches_lingo/models.py index fa87287a..6ce18cc9 100644 --- a/arches_lingo/models.py +++ b/arches_lingo/models.py @@ -2,6 +2,8 @@ from arches.app.models.models import ResourceInstance +from arches_lingo.querysets import LabelValueQuerySet + class VwLabelValue(models.Model): concept = models.ForeignKey( @@ -12,6 +14,8 @@ class VwLabelValue(models.Model): ) value = models.CharField(db_column="value") + objects = LabelValueQuerySet.as_manager() + class Meta: managed = False db_table = f"arches_lingo__vw_label_values" diff --git a/arches_lingo/query_utils.py b/arches_lingo/query_utils.py new file mode 100644 index 00000000..f8749a2e --- /dev/null +++ b/arches_lingo/query_utils.py @@ -0,0 +1,14 @@ +from django.db.models.expressions import Func + + +class JsonbArrayElements(Func): + """https://forum.djangoproject.com/t/django-4-2-behavior-change-when-using-arrayagg-on-unnested-arrayfield-postgresql-specific/21547/5""" + + arity = 1 + contains_subquery = True + function = "JSONB_ARRAY_ELEMENTS" + + +class LevenshteinLessEqual(Func): + arity = 3 + function = "LEVENSHTEIN_LESS_EQUAL" diff --git a/arches_lingo/querysets.py b/arches_lingo/querysets.py new file mode 100644 index 00000000..433085ae --- /dev/null +++ b/arches_lingo/querysets.py @@ -0,0 +1,23 @@ +from django.db import models + +from arches_lingo.query_utils import LevenshteinLessEqual + + +class LabelValueQuerySet(models.QuerySet): + + def fuzzy_search(self, term, max_edit_distance): + from arches_lingo.models import VwLabelValue + + return ( + VwLabelValue.objects.all() + .annotate( + edit_distance=LevenshteinLessEqual( + models.F("value"), + models.Value(term), + models.Value(max_edit_distance), + output_field=models.FloatField(), + ) + ) + .filter(edit_distance__lte=max_edit_distance) + .order_by("edit_distance") + ) diff --git a/arches_lingo/settings.py b/arches_lingo/settings.py index f9e7711e..fcef5756 100644 --- a/arches_lingo/settings.py +++ b/arches_lingo/settings.py @@ -165,7 +165,7 @@ "oauth2_provider.middleware.OAuth2TokenMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", "django.contrib.messages.middleware.MessageMiddleware", - # "django.middleware.clickjacking.XFrameOptionsMiddleware", + "django.middleware.clickjacking.XFrameOptionsMiddleware", "arches.app.utils.middleware.SetAnonymousUser", # "silk.middleware.SilkyMiddleware", ] diff --git a/arches_lingo/src/arches_lingo/App.vue b/arches_lingo/src/arches_lingo/App.vue index 292c84f5..4c8ad376 100644 --- a/arches_lingo/src/arches_lingo/App.vue +++ b/arches_lingo/src/arches_lingo/App.vue @@ -34,20 +34,13 @@ const { $gettext } = useGettext(); router.beforeEach(async (to, _from, next) => { try { - let userData = user.value; - - if (!userData || userData.username === ANONYMOUS) { - userData = await fetchUser(); - setUser(userData); - } + let userData = await fetchUser(); + setUser(userData); const requiresAuthentication = to.matched.some( (record) => record.meta.requiresAuthentication, ); - if ( - requiresAuthentication && - (!userData || userData.username === ANONYMOUS) - ) { + if (requiresAuthentication && userData.username === ANONYMOUS) { throw new Error(); } else { next(); diff --git a/arches_lingo/urls.py b/arches_lingo/urls.py index 1e5301af..f16faa89 100644 --- a/arches_lingo/urls.py +++ b/arches_lingo/urls.py @@ -3,7 +3,8 @@ from django.conf.urls.i18n import i18n_patterns from django.urls import include, path -from arches_lingo.views import LingoRootView, ConceptTreeView, ValueSearchView +from arches_lingo.views.root import LingoRootView +from arches_lingo.views.api.concepts import ConceptTreeView, ValueSearchView urlpatterns = [ path("", LingoRootView.as_view(), name="root"), @@ -11,7 +12,7 @@ path("search", LingoRootView.as_view(), name="search"), path("advanced-search", LingoRootView.as_view(), name="advanced-search"), path("schemes", LingoRootView.as_view(), name="schemes"), - path("api/concept_trees", ConceptTreeView.as_view(), name="concept_trees"), + path("api/concepts", ConceptTreeView.as_view(), name="api_concepts"), path("api/search", ValueSearchView.as_view(), name="api_search"), path("", include("arches_references.urls")), ] diff --git a/arches_lingo/views/api/concepts.py b/arches_lingo/views/api/concepts.py new file mode 100644 index 00000000..7c356ae8 --- /dev/null +++ b/arches_lingo/views/api/concepts.py @@ -0,0 +1,70 @@ +from django.core.paginator import Paginator +from django.utils.decorators import method_decorator +from django.views.generic import View + +from arches.app.models.system_settings import settings +from arches.app.utils.decorators import group_required +from arches.app.utils.response import JSONResponse + +from arches_lingo.models import VwLabelValue +from arches_lingo.concepts import ConceptBuilder + + +@method_decorator( + group_required("RDM Administrator", raise_exception=True), name="dispatch" +) +class ConceptTreeView(View): + def get(self, request): + builder = ConceptBuilder() + data = { + "schemes": [builder.serialize_scheme(scheme) for scheme in builder.schemes] + } + return JSONResponse(data) + + +@method_decorator( + group_required("RDM Administrator", raise_exception=True), name="dispatch" +) +class ValueSearchView(ConceptTreeView): + def get(self, request): + term = request.GET.get("term") + max_edit_distance = int( + request.GET.get("maxEditDistance", self.default_sensitivity()) + ) + page_number = request.GET.get("page", 1) + items_per_page = request.GET.get("items", 25) + + if term: + concept_query = VwLabelValue.objects.fuzzy_search(term, max_edit_distance) + else: + concept_query = VwLabelValue.objects.all().order_by("concept_id") + concept_query = concept_query.values_list("concept_id", flat=True).distinct() + + paginator = Paginator(concept_query, items_per_page) + page = paginator.get_page(page_number) + + data = [] + if page: + builder = ConceptBuilder() + data = [ + builder.serialize_concept( + str(concept_uuid), parents=True, children=False + ) + for concept_uuid in page + ] + + return JSONResponse(data) + + @staticmethod + def default_sensitivity(): + """Remains to be seen whether the existing elastic sensitivity setting + should be the fallback, but stub something out for now. + This sensitivity setting is actually inversely related to edit distance, + because it's prefix_length in elastic, not fuzziness, so invert it. + """ + elastic_prefix_length = settings.SEARCH_TERM_SENSITIVITY + if elastic_prefix_length <= 0: + return 5 + if elastic_prefix_length >= 5: + return 0 + return 5 - elastic_prefix_length diff --git a/tests/tests.py b/tests/tests.py index e067ed73..4ef30244 100644 --- a/tests/tests.py +++ b/tests/tests.py @@ -196,7 +196,7 @@ def test_get_concept_trees(self): # 4: select broader tiles, subquery for labels # 5: select top concept tiles, subquery for labels # 6: select schemes, subquery for labels - response = self.client.get(reverse("concept_trees")) + response = self.client.get(reverse("api_concepts")) self.assertEqual(response.status_code, 200) result = json.loads(response.content)