Reduce queries

arches/app/models/graph.py

@@ -699,7 +699,6 @@ def save(self, validate=True, nodeid=None):
                         ),
                         language=language,
                     )
-                    published_graph.save()
 
             # edge case for instantiating a serialized_graph that has a resource_instance_lifecycle not already in the system
             if self.resource_instance_lifecycle and not len(
@@ -2300,14 +2299,10 @@ def validate_fieldname(fieldname, fieldnames):
                 .filter(slug=self.slug)
             )
             if (
-                graphs_with_matching_slug.exists()
-                and graphs_with_matching_slug[0].graphid != self.graphid
-            ):
+                first_matching_graph := graphs_with_matching_slug.first()
+            ) and first_matching_graph.graphid != self.graphid:
                 if self.source_identifier_id:
-                    if (
-                        self.source_identifier_id
-                        != graphs_with_matching_slug[0].graphid
-                    ):
+                    if self.source_identifier_id != first_matching_graph.graphid:
                         raise GraphValidationError(
                             _(
                                 "Another resource model already uses the slug '{self.slug}'"
@@ -2372,14 +2367,14 @@ def update_published_graphs(self, user=None, notes=None):
                     elif len(published_graph_query) == 1:
                         published_graph = published_graph_query[0]
                         published_graph.serialized_graph = serialized_graph
+                        published_graph.save()
                     else:
                         raise GraphPublicationError(
                             message=_(
                                 "Multiple published graphs returned for language and publication_id"
                             )
                         )
 
-                    published_graph.save()
                     translation.deactivate()
 
     def create_editable_future_graph(self):
@@ -2935,7 +2930,6 @@ def publish(self, user=None, notes=None):
             publication = models.GraphXPublishedGraph.objects.create(
                 graph=self, notes=notes, user=user
             )
-            publication.save()
 
             self.publication = publication
             self.has_unpublished_changes = False
@@ -2947,16 +2941,14 @@ def publish(self, user=None, notes=None):
 
                 translation.activate(language=language_tuple[0])
 
-                published_graph = models.PublishedGraph.objects.create(
+                models.PublishedGraph.objects.create(
                     publication=publication,
                     serialized_graph=JSONDeserializer().deserialize(
                         JSONSerializer().serialize(self, force_recalculation=True)
                     ),
                     language=language,
                 )
 
-                published_graph.save()
-
             translation.deactivate()
 
 

arches/app/models/models.py

@@ -820,21 +820,21 @@ def is_collector(self):
         )
 
     def get_relatable_resources(self):
-        relatable_resource_ids = [
-            r2r.resourceclassfrom
-            for r2r in Resource2ResourceConstraint.objects.filter(
-                resourceclassto_id=self.nodeid
+        return [
+            (
+                constraint.resourceclassto
+                if constraint.resourceclassfrom_id == self.pk
+                else constraint.resourceclassfrom
             )
-            if r2r.resourceclassfrom is not None
-        ]
-        relatable_resource_ids = relatable_resource_ids + [
-            r2r.resourceclassto
-            for r2r in Resource2ResourceConstraint.objects.filter(
-                resourceclassfrom_id=self.nodeid
+            for constraint in (
+                self.resxres_contstraint_classes_from.filter(
+                    resourceclassto__isnull=False
+                )
+                | self.resxres_contstraint_classes_to.filter(
+                    resourceclassfrom__isnull=False
+                )
             )
-            if r2r.resourceclassto is not None
         ]
-        return relatable_resource_ids
 
     def set_relatable_resources(self, new_ids):
         old_ids = [res.nodeid for res in self.get_relatable_resources()]

arches/app/models/resource.py

@@ -331,7 +331,7 @@ def load_tiles(self, user=None, perm="read_nodegroup"):
             self.tiles = [
                 tile
                 for tile in self.tiles
-                if tile.nodegroup is not None
+                if tile.nodegroup_id is not None
                 and tile.nodegroup_id in readable_nodegroups
             ]
 

arches/app/permissions/arches_permission_base.py

@@ -767,7 +767,7 @@ def get_nodegroups_by_perm_for_user_or_group(
         NodeGroup,
     )
 
-    for nodegroup in NodeGroup.objects.all():
+    for nodegroup in NodeGroup.objects.only("nodegroupid").all():
         explicit_perms = checker.get_perms(nodegroup)
 
         if len(explicit_perms):

arches/app/search/components/search_results.py

@@ -144,7 +144,7 @@ def post_search_hook(self, search_query_object, response_object, **kwargs):
 
 def get_nodegroups_by_datatype_and_perm(request, datatype, permission):
     nodes = []
-    for node in Node.objects.filter(datatype=datatype):
+    for node in Node.objects.filter(datatype=datatype).select_related("nodegroup"):
         if request.user.has_perm(permission, node.nodegroup):
             nodes.append(str(node.nodegroup_id))
     return nodes

arches/app/search/components/time_filter.py

@@ -156,7 +156,7 @@ def view_data(self):
         date_datatypes = ["date", "edtf"]
         date_nodes = Node.objects.filter(
             datatype__in=date_datatypes, graph__isresource=True, graph__is_active=True
-        ).prefetch_related("nodegroup")
+        ).select_related("nodegroup")
         node_graph_dict = {
             str(node.nodeid): str(node.graph_id)
             for node in date_nodes

arches/app/search/search_export.py

@@ -80,11 +80,11 @@ def insert_subcard_below_parent_card(
     def return_ordered_header(self, graphid, export_type):
 
         subcard_list_with_sort = []
-        all_cards = models.CardModel.objects.filter(graph=graphid).prefetch_related(
+        all_cards = models.CardModel.objects.filter(graph=graphid).select_related(
             "nodegroup"
         )
-        all_card_list_with_sort = list(
-            all_cards.exclude(sortorder=None).order_by("sortorder")
+        all_card_list_with_sort = all_cards.exclude(sortorder=None).order_by(
+            "sortorder"
         )
         card_list_no_sort = list(all_cards.filter(sortorder=None))
         sorted_card_list = []
@@ -123,11 +123,10 @@ def order_cards(subcards_added=True):
 
         ordered_list_all_nodes = []
         for sorted_card in sorted_card_list:
-            card_node_objects = list(
-                models.CardXNodeXWidget.objects.filter(
-                    card_id=sorted_card.cardid
-                ).prefetch_related("node")
-            )
+            card_node_objects = models.CardXNodeXWidget.objects.filter(
+                card_id=sorted_card.cardid
+            ).select_related("node")
+
             if len(card_node_objects) > 0:
                 nodes_in_card = []
                 for card_node_object in card_node_objects:

arches/app/utils/index_database.py

@@ -248,43 +248,45 @@ def index_resources_using_singleprocessing(
         str(nodeid): datatype
         for nodeid, datatype in models.Node.objects.values_list("nodeid", "datatype")
     }
-    with se.BulkIndexer(batch_size=batch_size, refresh=True) as doc_indexer:
-        with se.BulkIndexer(batch_size=batch_size, refresh=True) as term_indexer:
-            if quiet is False:
-                if isinstance(resources, QuerySet):
-                    resource_count = resources.count()
-                else:
-                    resource_count = len(resources)
-                if resource_count > 1:
-                    bar = pyprind.ProgBar(resource_count, bar_char="█", title=title)
-                else:
-                    bar = None
-
-            for resource in optimize_resource_iteration(
-                resources, chunk_size=batch_size // 8
-            ):
-                resource.tiles = resource.prefetched_tiles
-                resource.descriptor_function = resource.graph.descriptor_function
-                resource.set_node_datatypes(node_datatypes)
-                resource.set_serialized_graph(get_serialized_graph(resource.graph))
-                if recalculate_descriptors:
-                    resource.save_descriptors()
-                if quiet is False and bar is not None:
-                    bar.update(item_id=resource)
-                document, terms = resource.get_documents_to_index(
-                    fetchTiles=False,
-                    datatype_factory=datatype_factory,
-                    node_datatypes=node_datatypes,
-                )
-                doc_indexer.add(
-                    index=RESOURCES_INDEX,
-                    id=document["resourceinstanceid"],
-                    data=document,
+    with (
+        se.BulkIndexer(batch_size=batch_size, refresh=True) as doc_indexer,
+        se.BulkIndexer(batch_size=batch_size, refresh=True) as term_indexer,
+    ):
+        if quiet is False:
+            if isinstance(resources, QuerySet):
+                resource_count = resources.count()
+            else:
+                resource_count = len(resources)
+            if resource_count > 1:
+                bar = pyprind.ProgBar(resource_count, bar_char="█", title=title)
+            else:
+                bar = None
+
+        for resource in optimize_resource_iteration(
+            resources, chunk_size=batch_size // 8
+        ):
+            resource.tiles = resource.prefetched_tiles
+            resource.descriptor_function = resource.graph.descriptor_function
+            resource.set_node_datatypes(node_datatypes)
+            resource.set_serialized_graph(get_serialized_graph(resource.graph))
+            if recalculate_descriptors:
+                resource.save_descriptors()
+            if quiet is False and bar is not None:
+                bar.update(item_id=resource)
+            document, terms = resource.get_documents_to_index(
+                fetchTiles=False,
+                datatype_factory=datatype_factory,
+                node_datatypes=node_datatypes,
+            )
+            doc_indexer.add(
+                index=RESOURCES_INDEX,
+                id=document["resourceinstanceid"],
+                data=document,
+            )
+            for term in terms:
+                term_indexer.add(
+                    index=TERMS_INDEX, id=term["_id"], data=term["_source"]
                 )
-                for term in terms:
-                    term_indexer.add(
-                        index=TERMS_INDEX, id=term["_id"], data=term["_source"]
-                    )
 
     return os.getpid()
 
@@ -325,16 +327,16 @@ def index_resources_by_type(
     for resource_type in resource_types:
         start = datetime.now()
 
-        graph_name = models.GraphModel.objects.get(graphid=str(resource_type)).name
+        graph_name = models.GraphModel.objects.get(graphid=resource_type).name
         logger.info("Indexing resource type '{0}'".format(graph_name))
 
         if clear_index:
             tq = Query(se=se)
-            cards = models.CardModel.objects.filter(
-                graph_id=str(resource_type)
-            ).select_related("nodegroup")
-            for nodegroup in [card.nodegroup for card in cards]:
-                term = Term(field="nodegroupid", term=str(nodegroup.nodegroupid))
+            cards = models.CardModel.objects.filter(graph_id=resource_type).only(
+                "nodegroup_id"
+            )
+            for card in cards:
+                term = Term(field="nodegroupid", term=str(card.nodegroup_id))
                 tq.add_query(term)
             tq.delete(index=TERMS_INDEX, refresh=True)
 
@@ -344,14 +346,11 @@ def index_resources_by_type(
             rq.delete(index=RESOURCES_INDEX, refresh=True)
 
         if use_multiprocessing:
-            resources = [
-                str(rid)
-                for rid in Resource.objects.filter(
-                    graph_id=str(resource_type)
-                ).values_list("resourceinstanceid", flat=True)
-            ]
+            resource_ids = models.ResourceInstance.objects.filter(
+                graph_id=resource_type
+            ).values_list("resourceinstanceid", flat=True)
             index_resources_using_multiprocessing(
-                resourceids=resources,
+                resourceids=resource_ids,
                 batch_size=batch_size,
                 quiet=quiet,
                 max_subprocesses=max_subprocesses,
@@ -363,7 +362,7 @@ def index_resources_by_type(
                 SearchEngineInstance as _se,
             )
 
-            resources = Resource.objects.filter(graph_id=str(resource_type))
+            resources = Resource.objects.filter(graph_id=resource_type)
             index_resources_using_singleprocessing(
                 resources=resources,
                 batch_size=batch_size,

arches/app/views/api/__init__.py

@@ -21,7 +21,7 @@
 from django.shortcuts import render
 from django.views.generic import View
 from django.db import transaction, connection
-from django.db.models import Q
+from django.db.models import Prefetch, Q
 from django.http import Http404, HttpResponse
 from django.http.request import QueryDict
 from django.core import management
@@ -1225,16 +1225,17 @@ def get(self, request, resourceid):
             cards = (
                 graph.cardmodel_set.order_by("sortorder")
                 .filter(nodegroup__in=nodegroups)
-                .prefetch_related("cardxnodexwidget_set")
+                .prefetch_related(
+                    Prefetch(
+                        "cardxnodexwidget_set",
+                        queryset=models.CardXNodeXWidget.objects.order_by("sortorder"),
+                    )
+                )
             )
             serialized_cards = JSONSerializer().serializeToPython(cards)
-            cardwidgets = [
-                widget
-                for widget in [
-                    card.cardxnodexwidget_set.order_by("sortorder").all()
-                    for card in cards
-                ]
-            ]
+            cardwidgets = []
+            for card in cards:
+                cardwidgets += card.cardxnodexwidget_set.all()
 
         editable_nodegroup_ids = [
             str(nodegroup.pk) for nodegroup in editable_nodegroups

arches/app/views/etl_manager.py

@@ -147,7 +147,7 @@ def get(self, request):
             all_events = (
                 LoadEvent.objects.all()
                 .order_by(("-load_start_time"))
-                .prefetch_related("user", "etl_module")
+                .select_related("user", "etl_module")
             )
             events = Paginator(all_events, item_per_page).page(page).object_list
             total = len(all_events)

arches/app/views/graph.py

@@ -87,22 +87,17 @@ def get(self, request, graphid):
             graphid=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID
         )
 
-        node_models = models.Node.objects.filter(
-            graph__pk__in=[resource_graph.pk for resource_graph in resource_graphs]
-        )
-
-        for res in resource_graphs:
-            try:
-                node_model = node_models.get(graph=res, istopnode=True)
-                resource_data.append(
-                    {
-                        "id": node_model.nodeid,
-                        "graph": res,
-                        "is_relatable": (node_model in relatable_resources),
-                    }
-                )
-            except models.Node.DoesNotExist:
-                pass
+        top_nodes = models.Node.objects.filter(
+            graph__in=resource_graphs, istopnode=True
+        ).select_related("graph")
+        for node_model in top_nodes:
+            resource_data.append(
+                {
+                    "id": node_model.nodeid,
+                    "graph": node_model.graph,
+                    "is_relatable": (node_model in relatable_resources),
+                }
+            )
 
         return JSONResponse(
             {