diff --git a/arches/app/models/graph.py b/arches/app/models/graph.py index 98cec8b4034..6155142efac 100644 --- a/arches/app/models/graph.py +++ b/arches/app/models/graph.py @@ -699,7 +699,6 @@ def save(self, validate=True, nodeid=None): ), language=language, ) - published_graph.save() # edge case for instantiating a serialized_graph that has a resource_instance_lifecycle not already in the system if self.resource_instance_lifecycle and not len( @@ -2300,14 +2299,10 @@ def validate_fieldname(fieldname, fieldnames): .filter(slug=self.slug) ) if ( - graphs_with_matching_slug.exists() - and graphs_with_matching_slug[0].graphid != self.graphid - ): + first_matching_graph := graphs_with_matching_slug.first() + ) and first_matching_graph.graphid != self.graphid: if self.source_identifier_id: - if ( - self.source_identifier_id - != graphs_with_matching_slug[0].graphid - ): + if self.source_identifier_id != first_matching_graph.graphid: raise GraphValidationError( _( "Another resource model already uses the slug '{self.slug}'" @@ -2372,6 +2367,7 @@ def update_published_graphs(self, user=None, notes=None): elif len(published_graph_query) == 1: published_graph = published_graph_query[0] published_graph.serialized_graph = serialized_graph + published_graph.save() else: raise GraphPublicationError( message=_( @@ -2379,7 +2375,6 @@ def update_published_graphs(self, user=None, notes=None): ) ) - published_graph.save() translation.deactivate() def create_editable_future_graph(self): @@ -2935,7 +2930,6 @@ def publish(self, user=None, notes=None): publication = models.GraphXPublishedGraph.objects.create( graph=self, notes=notes, user=user ) - publication.save() self.publication = publication self.has_unpublished_changes = False @@ -2947,7 +2941,7 @@ def publish(self, user=None, notes=None): translation.activate(language=language_tuple[0]) - published_graph = models.PublishedGraph.objects.create( + models.PublishedGraph.objects.create( publication=publication, serialized_graph=JSONDeserializer().deserialize( JSONSerializer().serialize(self, force_recalculation=True) @@ -2955,8 +2949,6 @@ def publish(self, user=None, notes=None): language=language, ) - published_graph.save() - translation.deactivate() diff --git a/arches/app/models/models.py b/arches/app/models/models.py index 73eec869e90..c7bad6a606e 100644 --- a/arches/app/models/models.py +++ b/arches/app/models/models.py @@ -820,21 +820,21 @@ def is_collector(self): ) def get_relatable_resources(self): - relatable_resource_ids = [ - r2r.resourceclassfrom - for r2r in Resource2ResourceConstraint.objects.filter( - resourceclassto_id=self.nodeid + return [ + ( + constraint.resourceclassto + if constraint.resourceclassfrom_id == self.pk + else constraint.resourceclassfrom ) - if r2r.resourceclassfrom is not None - ] - relatable_resource_ids = relatable_resource_ids + [ - r2r.resourceclassto - for r2r in Resource2ResourceConstraint.objects.filter( - resourceclassfrom_id=self.nodeid + for constraint in ( + self.resxres_contstraint_classes_from.filter( + resourceclassto__isnull=False + ) + | self.resxres_contstraint_classes_to.filter( + resourceclassfrom__isnull=False + ) ) - if r2r.resourceclassto is not None ] - return relatable_resource_ids def set_relatable_resources(self, new_ids): old_ids = [res.nodeid for res in self.get_relatable_resources()] diff --git a/arches/app/models/resource.py b/arches/app/models/resource.py index a53b3feefb0..fb261918982 100644 --- a/arches/app/models/resource.py +++ b/arches/app/models/resource.py @@ -331,7 +331,7 @@ def load_tiles(self, user=None, perm="read_nodegroup"): self.tiles = [ tile for tile in self.tiles - if tile.nodegroup is not None + if tile.nodegroup_id is not None and tile.nodegroup_id in readable_nodegroups ] diff --git a/arches/app/permissions/arches_permission_base.py b/arches/app/permissions/arches_permission_base.py index 3ee070c1533..3fd678cc54e 100644 --- a/arches/app/permissions/arches_permission_base.py +++ b/arches/app/permissions/arches_permission_base.py @@ -767,7 +767,7 @@ def get_nodegroups_by_perm_for_user_or_group( NodeGroup, ) - for nodegroup in NodeGroup.objects.all(): + for nodegroup in NodeGroup.objects.only("nodegroupid").all(): explicit_perms = checker.get_perms(nodegroup) if len(explicit_perms): diff --git a/arches/app/search/components/search_results.py b/arches/app/search/components/search_results.py index 94d72fecadf..36bfb22a4dd 100644 --- a/arches/app/search/components/search_results.py +++ b/arches/app/search/components/search_results.py @@ -144,7 +144,7 @@ def post_search_hook(self, search_query_object, response_object, **kwargs): def get_nodegroups_by_datatype_and_perm(request, datatype, permission): nodes = [] - for node in Node.objects.filter(datatype=datatype): + for node in Node.objects.filter(datatype=datatype).select_related("nodegroup"): if request.user.has_perm(permission, node.nodegroup): nodes.append(str(node.nodegroup_id)) return nodes diff --git a/arches/app/search/components/time_filter.py b/arches/app/search/components/time_filter.py index 6413d125ad8..522f059c20a 100644 --- a/arches/app/search/components/time_filter.py +++ b/arches/app/search/components/time_filter.py @@ -156,7 +156,7 @@ def view_data(self): date_datatypes = ["date", "edtf"] date_nodes = Node.objects.filter( datatype__in=date_datatypes, graph__isresource=True, graph__is_active=True - ).prefetch_related("nodegroup") + ).select_related("nodegroup") node_graph_dict = { str(node.nodeid): str(node.graph_id) for node in date_nodes diff --git a/arches/app/search/search_export.py b/arches/app/search/search_export.py index 2a3fc0d4754..8e23e432ade 100644 --- a/arches/app/search/search_export.py +++ b/arches/app/search/search_export.py @@ -80,11 +80,11 @@ def insert_subcard_below_parent_card( def return_ordered_header(self, graphid, export_type): subcard_list_with_sort = [] - all_cards = models.CardModel.objects.filter(graph=graphid).prefetch_related( + all_cards = models.CardModel.objects.filter(graph=graphid).select_related( "nodegroup" ) - all_card_list_with_sort = list( - all_cards.exclude(sortorder=None).order_by("sortorder") + all_card_list_with_sort = all_cards.exclude(sortorder=None).order_by( + "sortorder" ) card_list_no_sort = list(all_cards.filter(sortorder=None)) sorted_card_list = [] @@ -123,11 +123,10 @@ def order_cards(subcards_added=True): ordered_list_all_nodes = [] for sorted_card in sorted_card_list: - card_node_objects = list( - models.CardXNodeXWidget.objects.filter( - card_id=sorted_card.cardid - ).prefetch_related("node") - ) + card_node_objects = models.CardXNodeXWidget.objects.filter( + card_id=sorted_card.cardid + ).select_related("node") + if len(card_node_objects) > 0: nodes_in_card = [] for card_node_object in card_node_objects: diff --git a/arches/app/utils/index_database.py b/arches/app/utils/index_database.py index c0d14bd0f17..c140483a2e6 100644 --- a/arches/app/utils/index_database.py +++ b/arches/app/utils/index_database.py @@ -248,43 +248,45 @@ def index_resources_using_singleprocessing( str(nodeid): datatype for nodeid, datatype in models.Node.objects.values_list("nodeid", "datatype") } - with se.BulkIndexer(batch_size=batch_size, refresh=True) as doc_indexer: - with se.BulkIndexer(batch_size=batch_size, refresh=True) as term_indexer: - if quiet is False: - if isinstance(resources, QuerySet): - resource_count = resources.count() - else: - resource_count = len(resources) - if resource_count > 1: - bar = pyprind.ProgBar(resource_count, bar_char="█", title=title) - else: - bar = None - - for resource in optimize_resource_iteration( - resources, chunk_size=batch_size // 8 - ): - resource.tiles = resource.prefetched_tiles - resource.descriptor_function = resource.graph.descriptor_function - resource.set_node_datatypes(node_datatypes) - resource.set_serialized_graph(get_serialized_graph(resource.graph)) - if recalculate_descriptors: - resource.save_descriptors() - if quiet is False and bar is not None: - bar.update(item_id=resource) - document, terms = resource.get_documents_to_index( - fetchTiles=False, - datatype_factory=datatype_factory, - node_datatypes=node_datatypes, - ) - doc_indexer.add( - index=RESOURCES_INDEX, - id=document["resourceinstanceid"], - data=document, + with ( + se.BulkIndexer(batch_size=batch_size, refresh=True) as doc_indexer, + se.BulkIndexer(batch_size=batch_size, refresh=True) as term_indexer, + ): + if quiet is False: + if isinstance(resources, QuerySet): + resource_count = resources.count() + else: + resource_count = len(resources) + if resource_count > 1: + bar = pyprind.ProgBar(resource_count, bar_char="█", title=title) + else: + bar = None + + for resource in optimize_resource_iteration( + resources, chunk_size=batch_size // 8 + ): + resource.tiles = resource.prefetched_tiles + resource.descriptor_function = resource.graph.descriptor_function + resource.set_node_datatypes(node_datatypes) + resource.set_serialized_graph(get_serialized_graph(resource.graph)) + if recalculate_descriptors: + resource.save_descriptors() + if quiet is False and bar is not None: + bar.update(item_id=resource) + document, terms = resource.get_documents_to_index( + fetchTiles=False, + datatype_factory=datatype_factory, + node_datatypes=node_datatypes, + ) + doc_indexer.add( + index=RESOURCES_INDEX, + id=document["resourceinstanceid"], + data=document, + ) + for term in terms: + term_indexer.add( + index=TERMS_INDEX, id=term["_id"], data=term["_source"] ) - for term in terms: - term_indexer.add( - index=TERMS_INDEX, id=term["_id"], data=term["_source"] - ) return os.getpid() @@ -325,16 +327,16 @@ def index_resources_by_type( for resource_type in resource_types: start = datetime.now() - graph_name = models.GraphModel.objects.get(graphid=str(resource_type)).name + graph_name = models.GraphModel.objects.get(graphid=resource_type).name logger.info("Indexing resource type '{0}'".format(graph_name)) if clear_index: tq = Query(se=se) - cards = models.CardModel.objects.filter( - graph_id=str(resource_type) - ).select_related("nodegroup") - for nodegroup in [card.nodegroup for card in cards]: - term = Term(field="nodegroupid", term=str(nodegroup.nodegroupid)) + cards = models.CardModel.objects.filter(graph_id=resource_type).only( + "nodegroup_id" + ) + for card in cards: + term = Term(field="nodegroupid", term=str(card.nodegroup_id)) tq.add_query(term) tq.delete(index=TERMS_INDEX, refresh=True) @@ -344,14 +346,11 @@ def index_resources_by_type( rq.delete(index=RESOURCES_INDEX, refresh=True) if use_multiprocessing: - resources = [ - str(rid) - for rid in Resource.objects.filter( - graph_id=str(resource_type) - ).values_list("resourceinstanceid", flat=True) - ] + resource_ids = models.ResourceInstance.objects.filter( + graph_id=resource_type + ).values_list("resourceinstanceid", flat=True) index_resources_using_multiprocessing( - resourceids=resources, + resourceids=resource_ids, batch_size=batch_size, quiet=quiet, max_subprocesses=max_subprocesses, @@ -363,7 +362,7 @@ def index_resources_by_type( SearchEngineInstance as _se, ) - resources = Resource.objects.filter(graph_id=str(resource_type)) + resources = Resource.objects.filter(graph_id=resource_type) index_resources_using_singleprocessing( resources=resources, batch_size=batch_size, diff --git a/arches/app/views/api/__init__.py b/arches/app/views/api/__init__.py index 1e49a5b18f6..070122c4ee6 100644 --- a/arches/app/views/api/__init__.py +++ b/arches/app/views/api/__init__.py @@ -21,7 +21,7 @@ from django.shortcuts import render from django.views.generic import View from django.db import transaction, connection -from django.db.models import Q +from django.db.models import Prefetch, Q from django.http import Http404, HttpResponse from django.http.request import QueryDict from django.core import management @@ -1225,16 +1225,17 @@ def get(self, request, resourceid): cards = ( graph.cardmodel_set.order_by("sortorder") .filter(nodegroup__in=nodegroups) - .prefetch_related("cardxnodexwidget_set") + .prefetch_related( + Prefetch( + "cardxnodexwidget_set", + queryset=models.CardXNodeXWidget.objects.order_by("sortorder"), + ) + ) ) serialized_cards = JSONSerializer().serializeToPython(cards) - cardwidgets = [ - widget - for widget in [ - card.cardxnodexwidget_set.order_by("sortorder").all() - for card in cards - ] - ] + cardwidgets = [] + for card in cards: + cardwidgets += card.cardxnodexwidget_set.all() editable_nodegroup_ids = [ str(nodegroup.pk) for nodegroup in editable_nodegroups diff --git a/arches/app/views/etl_manager.py b/arches/app/views/etl_manager.py index 554d01a1511..d451a5f531f 100644 --- a/arches/app/views/etl_manager.py +++ b/arches/app/views/etl_manager.py @@ -147,7 +147,7 @@ def get(self, request): all_events = ( LoadEvent.objects.all() .order_by(("-load_start_time")) - .prefetch_related("user", "etl_module") + .select_related("user", "etl_module") ) events = Paginator(all_events, item_per_page).page(page).object_list total = len(all_events) diff --git a/arches/app/views/graph.py b/arches/app/views/graph.py index 660b0221970..bd5b1b7719c 100644 --- a/arches/app/views/graph.py +++ b/arches/app/views/graph.py @@ -87,22 +87,17 @@ def get(self, request, graphid): graphid=settings.SYSTEM_SETTINGS_RESOURCE_MODEL_ID ) - node_models = models.Node.objects.filter( - graph__pk__in=[resource_graph.pk for resource_graph in resource_graphs] - ) - - for res in resource_graphs: - try: - node_model = node_models.get(graph=res, istopnode=True) - resource_data.append( - { - "id": node_model.nodeid, - "graph": res, - "is_relatable": (node_model in relatable_resources), - } - ) - except models.Node.DoesNotExist: - pass + top_nodes = models.Node.objects.filter( + graph__in=resource_graphs, istopnode=True + ).select_related("graph") + for node_model in top_nodes: + resource_data.append( + { + "id": node_model.nodeid, + "graph": node_model.graph, + "is_relatable": (node_model in relatable_resources), + } + ) return JSONResponse( { diff --git a/arches/app/views/resource.py b/arches/app/views/resource.py index 5a0f4947e78..6f61ad6da18 100644 --- a/arches/app/views/resource.py +++ b/arches/app/views/resource.py @@ -22,6 +22,7 @@ from django.core.exceptions import ObjectDoesNotExist, PermissionDenied from django.contrib.auth.models import User, Group +from django.db.models import Prefetch from django.forms.models import model_to_dict from django.http import HttpResponseNotFound from django.http import Http404 @@ -303,14 +304,17 @@ def add_i18n_to_widget_defaults(widgets): cards = ( graph.cardmodel_set.order_by("sortorder") .filter(nodegroup__in=nodegroups) - .prefetch_related("cardxnodexwidget_set") + .prefetch_related( + Prefetch( + "cardxnodexwidget_set", + queryset=models.CardXNodeXWidget.objects.order_by("sortorder"), + ) + ) ) serialized_cards = JSONSerializer().serializeToPython(cards) cardwidgets = [] for card in cards: - cardwidgets += list( - card.cardxnodexwidget_set.order_by("sortorder").all() - ) + cardwidgets += card.cardxnodexwidget_set.all() updated_cardwidgets = add_i18n_to_cardwidget_defaults(cardwidgets)