From 41e081cdb7939967220086f2d397892ed4be4264 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Novotn=C3=BD?= Date: Tue, 13 Feb 2024 17:08:59 +0100 Subject: [PATCH] fix(#471): Extra results sorter does not carry the correct query context Query that used prefetch evaluation logic revealed that sorters in extra result producers - namely facet summary and hierarchy summary stick to the original query context. This resulted in incorrect sorting implementation which tried to locate entities of invalid type in original queried entity query context, which provided unexpected NULL value. --- .../java/io/evitadb/core/SessionRegistry.java | 8 +- .../io/evitadb/core/query/QueryContext.java | 16 ++ .../io/evitadb/core/query/QueryPlanner.java | 2 +- .../prefetch/PrefetchFormulaVisitor.java | 7 +- .../ExtraResultPlanningVisitor.java | 97 +++++++---- .../FacetSummaryOfReferenceTranslator.java | 22 +-- .../facet/producer/FacetSummaryProducer.java | 151 ++++++++++-------- .../HierarchyOfReferenceTranslator.java | 13 +- .../HierarchyOfSelfTranslator.java | 13 +- .../producer/HierarchySet.java | 13 +- .../producer/HierarchyStatisticsProducer.java | 8 +- .../core/query/sort/ConditionalSorter.java | 2 +- .../core/query/sort/NestedContextSorter.java | 58 +++++++ .../http/ExternalApiExceptionHandler.java | 4 +- 14 files changed, 261 insertions(+), 153 deletions(-) create mode 100644 evita_engine/src/main/java/io/evitadb/core/query/sort/NestedContextSorter.java diff --git a/evita_engine/src/main/java/io/evitadb/core/SessionRegistry.java b/evita_engine/src/main/java/io/evitadb/core/SessionRegistry.java index ac1b67827..3364fe220 100644 --- a/evita_engine/src/main/java/io/evitadb/core/SessionRegistry.java +++ b/evita_engine/src/main/java/io/evitadb/core/SessionRegistry.java @@ -178,15 +178,13 @@ public Object invoke(Object proxy, Method method, Object[] args) { throw evitaInvalidUsageException; } else if (targetException instanceof EvitaInternalError evitaInternalError) { log.error( - "Internal Evita error occurred in {}: {}", - evitaInternalError.getErrorCode(), - evitaInternalError.getPrivateMessage(), + "Internal Evita error occurred in " + evitaInternalError.getErrorCode() + ": " + evitaInternalError.getPrivateMessage(), targetException ); // unwrap and rethrow throw evitaInternalError; } else { - log.error("Unexpected internal Evita error occurred: {}", ex.getCause().getMessage(), targetException); + log.error("Unexpected internal Evita error occurred: " + ex.getCause().getMessage(), targetException); throw new EvitaInternalError( "Unexpected internal Evita error occurred: " + ex.getCause().getMessage(), "Unexpected internal Evita error occurred.", @@ -194,7 +192,7 @@ public Object invoke(Object proxy, Method method, Object[] args) { ); } } catch (Throwable ex) { - log.error("Unexpected system error occurred: {}", ex.getMessage(), ex); + log.error("Unexpected system error occurred: " + ex.getMessage(), ex); throw new EvitaInternalError( "Unexpected system error occurred: " + ex.getMessage(), "Unexpected system error occurred.", diff --git a/evita_engine/src/main/java/io/evitadb/core/query/QueryContext.java b/evita_engine/src/main/java/io/evitadb/core/query/QueryContext.java index 0680fa122..9cc0b6111 100644 --- a/evita_engine/src/main/java/io/evitadb/core/query/QueryContext.java +++ b/evita_engine/src/main/java/io/evitadb/core/query/QueryContext.java @@ -908,6 +908,22 @@ public SealedEntity enrichOrLimitReferencedEntity( PRIVATE METHODS */ + /** + * Method returns appropriate {@link EntityCollection} for the {@link #evitaRequest} or empty value. + */ + @Nonnull + public Optional getEntityCollection(@Nullable String entityType) { + if (entityType == null) { + return Optional.empty(); + } else if (Objects.equals(entityType, this.entityType) && entityCollection != null) { + return Optional.of(entityCollection); + } else { + return Optional.ofNullable( + (EntityCollection) catalog.getCollectionForEntity(entityType).orElse(null) + ); + } + } + /** * Method returns appropriate {@link EntityCollection} for the {@link #evitaRequest} or throws comprehensible * exception. In order exception to be comprehensible you need to provide sensible `reason` for accessing diff --git a/evita_engine/src/main/java/io/evitadb/core/query/QueryPlanner.java b/evita_engine/src/main/java/io/evitadb/core/query/QueryPlanner.java index 3f68fb288..a47924f0b 100644 --- a/evita_engine/src/main/java/io/evitadb/core/query/QueryPlanner.java +++ b/evita_engine/src/main/java/io/evitadb/core/query/QueryPlanner.java @@ -413,7 +413,7 @@ private static PrefetchFormulaVisitor createPrefetchFormulaVisitor( @Nonnull QueryContext queryContext ) { if (targetIndex.isGlobalIndex() || targetIndex.isCatalogIndex()) { - return new PrefetchFormulaVisitor(queryContext); + return new PrefetchFormulaVisitor(); } else { return null; } diff --git a/evita_engine/src/main/java/io/evitadb/core/query/algebra/prefetch/PrefetchFormulaVisitor.java b/evita_engine/src/main/java/io/evitadb/core/query/algebra/prefetch/PrefetchFormulaVisitor.java index 6a2fc5a20..08cadf16f 100644 --- a/evita_engine/src/main/java/io/evitadb/core/query/algebra/prefetch/PrefetchFormulaVisitor.java +++ b/evita_engine/src/main/java/io/evitadb/core/query/algebra/prefetch/PrefetchFormulaVisitor.java @@ -76,10 +76,6 @@ public class PrefetchFormulaVisitor implements FormulaVisitor, FormulaPostProces * that needs to be prefetched. */ @Nonnull private final Bitmap entityReferences = new BaseBitmap(); - /** - * The query context that will be used to prefetch entities. - */ - @Nonnull private final QueryContext queryContext; /** * Flag that signalizes {@link #visit(Formula)} happens in conjunctive scope. */ @@ -134,8 +130,7 @@ private static long estimatePrefetchCost(int prefetchedEntityCount, @Nonnull Ent return PREFETCH_COST_ESTIMATOR.apply(prefetchedEntityCount, requirements.getRequirements().length); } - public PrefetchFormulaVisitor(@Nonnull QueryContext queryContext) { - this.queryContext = queryContext; + public PrefetchFormulaVisitor() { } /** diff --git a/evita_engine/src/main/java/io/evitadb/core/query/extraResult/ExtraResultPlanningVisitor.java b/evita_engine/src/main/java/io/evitadb/core/query/extraResult/ExtraResultPlanningVisitor.java index f0acf4641..c9983a8cb 100644 --- a/evita_engine/src/main/java/io/evitadb/core/query/extraResult/ExtraResultPlanningVisitor.java +++ b/evita_engine/src/main/java/io/evitadb/core/query/extraResult/ExtraResultPlanningVisitor.java @@ -36,6 +36,7 @@ import io.evitadb.api.query.filter.HierarchyWithinRoot; import io.evitadb.api.query.filter.ReferenceHaving; import io.evitadb.api.query.filter.UserFilter; +import io.evitadb.api.query.order.OrderBy; import io.evitadb.api.query.require.*; import io.evitadb.api.query.visitor.ConstraintCloneVisitor; import io.evitadb.api.requestResponse.EvitaRequest; @@ -43,6 +44,7 @@ import io.evitadb.api.requestResponse.extraResult.QueryTelemetry.QueryPhase; import io.evitadb.api.requestResponse.schema.EntitySchemaContract; import io.evitadb.api.requestResponse.schema.ReferenceSchemaContract; +import io.evitadb.core.EntityCollection; import io.evitadb.core.query.AttributeSchemaAccessor; import io.evitadb.core.query.PrefetchRequirementCollector; import io.evitadb.core.query.QueryContext; @@ -74,11 +76,14 @@ import io.evitadb.core.query.extraResult.translator.reference.ReferenceContentTranslator; import io.evitadb.core.query.indexSelection.TargetIndexes; import io.evitadb.core.query.sort.DeferredSorter; +import io.evitadb.core.query.sort.NestedContextSorter; +import io.evitadb.core.query.sort.NoSorter; import io.evitadb.core.query.sort.OrderByVisitor; import io.evitadb.core.query.sort.Sorter; import io.evitadb.core.query.sort.attribute.translator.EntityAttributeExtractor; import io.evitadb.exception.EvitaInternalError; import io.evitadb.index.EntityIndex; +import io.evitadb.index.GlobalEntityIndex; import io.evitadb.utils.ArrayUtils; import lombok.Getter; import lombok.experimental.Delegate; @@ -164,6 +169,10 @@ public class ExtraResultPlanningVisitor implements ConstraintVisitor { * Contains an accessor providing access to the attribute schemas. */ @Getter private final AttributeSchemaAccessor attributeSchemaAccessor; + /** + * Contemporary stack for auxiliary data resolved for each level of the query. + */ + private final Deque scope = new ArrayDeque<>(32); /** * Performance optimization when multiple translators ask for the same (last) producer. */ @@ -189,10 +198,6 @@ public class ExtraResultPlanningVisitor implements ConstraintVisitor { * times. */ private Set userFilterFormula; - /** - * Contemporary stack for auxiliary data resolved for each level of the query. - */ - private final Deque scope = new ArrayDeque<>(32); public ExtraResultPlanningVisitor( @Nonnull QueryContext queryContext, @@ -357,10 +362,10 @@ public FilterBy getFilterByWithoutHierarchyAndUserFilter(@Nullable ReferenceSche * the {@link io.evitadb.api.requestResponse.extraResult.Hierarchy} result object. */ @Nonnull - public Sorter createSorter( + public NestedContextSorter createSorter( @Nonnull ConstraintContainer orderBy, @Nullable Locale locale, - @Nonnull EntityIndex entityIndex, + @Nonnull EntityCollection entityCollection, @Nonnull String entityType, @Nonnull Supplier stepDescriptionSupplier ) { @@ -369,38 +374,62 @@ public Sorter createSorter( QueryPhase.PLANNING_SORT, stepDescriptionSupplier ); - // crete a visitor - final OrderByVisitor orderByVisitor = new OrderByVisitor( - queryContext, - Collections.emptyList(), - prefetchRequirementCollector, - filteringFormula - ); - // now analyze the filter by in a nested context with exchanged primary entity index - return orderByVisitor.executeInContext( - new EntityIndex[] {entityIndex}, - entityType, - locale, - new AttributeSchemaAccessor(queryContext.getCatalogSchema(), queryContext.getSchema(entityType)), - EntityAttributeExtractor.INSTANCE, - () -> { - for (OrderConstraint innerConstraint : orderBy.getChildren()) { - innerConstraint.accept(orderByVisitor); - } - // create a deferred sorter that will log the execution time to query telemetry - return new DeferredSorter( - orderByVisitor.getSorter(), - sorter -> { - try { - queryContext.pushStep(QueryPhase.EXECUTION_SORT_AND_SLICE, stepDescriptionSupplier); - return sorter.getAsInt(); - } finally { - queryContext.popStep(); + // we have to create and trap the nested query context here to carry it along with the sorter + // otherwise the sorter will target and use the incorrectly originally queried (prefetched) entities + try ( + final QueryContext nestedQueryContext = entityCollection.createQueryContext( + queryContext, + queryContext.getEvitaRequest().deriveCopyWith( + entityType, + null, + new OrderBy(orderBy.getChildren()), + queryContext.getLocale() + ), + queryContext.getEvitaSession() + ) + ) { + final GlobalEntityIndex entityIndex = entityCollection.getGlobalIndexIfExists().orElse(null); + final Sorter sorter; + if (entityIndex == null) { + sorter = NoSorter.INSTANCE; + } else { + // create a visitor + final OrderByVisitor orderByVisitor = new OrderByVisitor( + nestedQueryContext, + Collections.emptyList(), + prefetchRequirementCollector, + filteringFormula + ); + // now analyze the filter by in a nested context with exchanged primary entity index + sorter = orderByVisitor.executeInContext( + new EntityIndex[]{entityIndex}, + entityType, + locale, + new AttributeSchemaAccessor(nestedQueryContext.getCatalogSchema(), entityCollection.getSchema()), + EntityAttributeExtractor.INSTANCE, + () -> { + for (OrderConstraint innerConstraint : orderBy.getChildren()) { + innerConstraint.accept(orderByVisitor); } + // create a deferred sorter that will log the execution time to query telemetry + return new DeferredSorter( + orderByVisitor.getSorter(), + theSorter -> { + try { + nestedQueryContext.pushStep(QueryPhase.EXECUTION_SORT_AND_SLICE, stepDescriptionSupplier); + return theSorter.getAsInt(); + } finally { + nestedQueryContext.popStep(); + } + } + ); } ); } - ); + return new NestedContextSorter( + nestedQueryContext, sorter + ); + } } finally { queryContext.popStep(); } diff --git a/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/facet/FacetSummaryOfReferenceTranslator.java b/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/facet/FacetSummaryOfReferenceTranslator.java index 2aa76b56f..ac9da6fd7 100644 --- a/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/facet/FacetSummaryOfReferenceTranslator.java +++ b/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/facet/FacetSummaryOfReferenceTranslator.java @@ -52,8 +52,8 @@ import io.evitadb.core.query.extraResult.translator.facet.producer.FilteringFormulaPredicate; import io.evitadb.core.query.extraResult.translator.reference.EntityFetchTranslator; import io.evitadb.core.query.indexSelection.TargetIndexes; +import io.evitadb.core.query.sort.NestedContextSorter; import io.evitadb.core.query.sort.NoSorter; -import io.evitadb.core.query.sort.Sorter; import io.evitadb.index.EntityIndex; import io.evitadb.index.bitmap.Bitmap; import io.evitadb.index.bitmap.collection.BitmapIntoBitmapCollector; @@ -159,7 +159,7 @@ static IntPredicate createFacetPredicate( * @return the created facet sorter, or null if the reference schema is not managed and sorting is not required */ @Nullable - static Sorter createFacetSorter( + static NestedContextSorter createFacetSorter( @Nonnull OrderBy orderBy, @Nullable Locale locale, @Nonnull ExtraResultPlanningVisitor extraResultPlanner, @@ -175,16 +175,16 @@ static Sorter createFacetSorter( } else if (!referenceSchema.isReferencedEntityTypeManaged()) { return null; } - return extraResultPlanner.getGlobalEntityIndexIfExists(referenceSchema.getReferencedEntityType()) - .map(ix -> extraResultPlanner.createSorter( + return extraResultPlanner.getEntityCollection(referenceSchema.getReferencedEntityType()) + .map(collection -> extraResultPlanner.createSorter( orderBy, locale, - ix, + collection, referenceSchema.getReferencedEntityType(), () -> "Facet summary `" + referenceSchema.getName() + "` facet ordering: " + orderBy ) ) - .orElse(NoSorter.INSTANCE); + .orElseGet(() -> new NestedContextSorter(extraResultPlanner.getQueryContext(), NoSorter.INSTANCE)); } /** @@ -198,7 +198,7 @@ static Sorter createFacetSorter( * @return The created sorter for facet group ordering, or null if not required. */ @Nullable - static Sorter createFacetGroupSorter( + static NestedContextSorter createFacetGroupSorter( @Nullable OrderGroupBy orderBy, @Nullable Locale locale, @Nonnull ExtraResultPlanningVisitor extraResultPlanner, @@ -215,16 +215,16 @@ static Sorter createFacetGroupSorter( return null; } - return extraResultPlanner.getGlobalEntityIndexIfExists(referenceSchema.getReferencedGroupType()) - .map(ix -> extraResultPlanner.createSorter( + return extraResultPlanner.getEntityCollection(referenceSchema.getReferencedGroupType()) + .map(collection -> extraResultPlanner.createSorter( orderBy, locale, - ix, + collection, referenceSchema.getReferencedGroupType(), () -> "Facet summary `" + referenceSchema.getName() + "` group ordering: " + orderBy ) ) - .orElse(NoSorter.INSTANCE); + .orElseGet(() -> new NestedContextSorter(extraResultPlanner.getQueryContext(), NoSorter.INSTANCE)); } /** diff --git a/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/facet/producer/FacetSummaryProducer.java b/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/facet/producer/FacetSummaryProducer.java index a2f76d55e..549459bb6 100644 --- a/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/facet/producer/FacetSummaryProducer.java +++ b/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/facet/producer/FacetSummaryProducer.java @@ -44,7 +44,7 @@ import io.evitadb.core.query.algebra.base.ConstantFormula; import io.evitadb.core.query.algebra.base.OrFormula; import io.evitadb.core.query.extraResult.ExtraResultProducer; -import io.evitadb.core.query.sort.Sorter; +import io.evitadb.core.query.sort.NestedContextSorter; import io.evitadb.core.query.sort.utils.SortUtils; import io.evitadb.exception.EvitaInternalError; import io.evitadb.index.bitmap.BaseBitmap; @@ -160,14 +160,14 @@ public FacetSummaryProducer( /** * Registers default settings for facet summary in terms of entity richness (both group and facet) and also * a default type of statistics depth. These settings will be used for all facet references that are not explicitly - * configured by {@link #requireReferenceFacetSummary(ReferenceSchemaContract, FacetStatisticsDepth, IntPredicate, IntPredicate, Sorter, Sorter, EntityFetch, EntityGroupFetch)}. + * configured by {@link #requireReferenceFacetSummary(ReferenceSchemaContract, FacetStatisticsDepth, IntPredicate, IntPredicate, NestedContextSorter, NestedContextSorter, EntityFetch, EntityGroupFetch)}. */ public void requireDefaultFacetSummary( @Nonnull FacetStatisticsDepth facetStatisticsDepth, @Nullable Function facetPredicate, @Nullable Function groupPredicate, - @Nullable Function facetSorter, - @Nullable Function groupSorter, + @Nullable Function facetSorter, + @Nullable Function groupSorter, @Nullable EntityFetch facetEntityRequirement, @Nullable EntityGroupFetch groupEntityRequirement ) { @@ -191,8 +191,8 @@ public void requireReferenceFacetSummary( @Nonnull FacetStatisticsDepth facetStatisticsDepth, @Nullable IntPredicate facetPredicate, @Nullable IntPredicate groupPredicate, - @Nullable Sorter facetSorter, - @Nullable Sorter groupSorter, + @Nullable NestedContextSorter facetSorter, + @Nullable NestedContextSorter groupSorter, @Nullable EntityFetch facetEntityRequirement, @Nullable EntityGroupFetch groupEntityRequirement ) { @@ -235,7 +235,6 @@ public EvitaResponseExtraResult fabricate(@Nonnull List Collectors.mapping( Function.identity(), new FacetGroupStatisticsCollector( - queryContext, // translates Facet#type to EntitySchema#reference#groupType referenceName -> queryContext.getSchema().getReferenceOrThrowException(referenceName), referenceSchema -> ofNullable(facetSummaryRequests.get(referenceSchema.getName())) @@ -313,10 +312,6 @@ private BiFunction createFetcherFunction(@Nul */ @RequiredArgsConstructor private static class FacetGroupStatisticsCollector implements Collector, Collection> { - /** - * The query context used for querying the entities. - */ - private final QueryContext queryContext; /** * Translates {@link FacetHaving#getReferenceName()} to {@link EntitySchema#getReference(String)}. */ @@ -493,6 +488,80 @@ private static Map> getGroupEntitiesIndex ); } + /** + * This method takes a map of facet statistics and a nested context sorter, and returns an array of sorted facet primary keys. + * + * @param theFacetStatistics map of facet statistics, where the key is the facet primary key and the value is the facet accumulator + * @param sorter nested context sorter used for sorting the facets + * @return array of sorted facet primary keys + */ + @Nonnull + private static int[] getSortedFacets(@Nonnull Map theFacetStatistics, @Nonnull NestedContextSorter sorter) { + // if the sorter is defined, sort them + final RoaringBitmapWriter writer = RoaringBitmapBackedBitmap.buildWriter(); + // collect all entity primary keys + theFacetStatistics.keySet().forEach(writer::add); + // create sorted array using the sorter + final ConstantFormula unsortedIds = new ConstantFormula(new BaseBitmap(writer.get())); + final Bitmap recordsToSort = unsortedIds.compute(); + final int count = recordsToSort.size(); + final int[] result = new int[count]; + final int peak = sorter.sortAndSlice(unsortedIds, 0, count, result, 0); + return SortUtils.asResult(result, peak); + } + + /** + * Compares two {@link GroupAccumulator} objects based on their facet group summaries. + * The comparison logic is as follows: + * 1. If the reference schema of o1 and o2 are different, compare based on the order of their facet summary requests. + * 2. If the facet summary request of o1 has a group sorter defined, the facet group summaries are sorted using the sorter. + * The sorted group summaries are then used to determine the order of o1 and o2 based on their group ids. + * 3. If the facet summary request of o2 has a group sorter defined, the facet group summaries are sorted using the sorter. + * The sorted group summaries are then used to determine the order of o1 and o2 based on their group ids. + * 4. If neither o1 or o2 have a group sorter defined and o1's group id is null, o1 is considered greater than o2. + * 5. If neither o1 or o2 have a group sorter defined and o2's group id is null, o1 is considered less than o2. + * 6. If neither o1 or o2 have a group sorter defined and both o1 and o2 have group ids, compare based on their group ids. + * + * @param groupIdIndex the index of facet groups by reference name + * @param sortedGroupIds the sorted group ids by reference name + * @param o1 the first GroupAccumulator object to compare + * @param o2 the second GroupAccumulator object to compare + * @return a negative integer, zero, or a positive integer as o1 is less than, equal to, or greater than o2 + */ + private static int compareFacetGroupSummaries(@Nonnull Map groupIdIndex, @Nonnull Map sortedGroupIds, @Nonnull GroupAccumulator o1, @Nonnull GroupAccumulator o2) { + if (o1.getReferenceSchema() != o2.getReferenceSchema()) { + return Integer.compare(o1.getFacetSummaryRequest().order(), o2.getFacetSummaryRequest().order()); + } else if (o1.getFacetSummaryRequest().groupSorter() != null) { + final NestedContextSorter sorter = o1.getFacetSummaryRequest().groupSorter(); + // create sorted array using the sorter + final String referenceName = o1.getFacetSummaryRequest().referenceSchema().getName(); + final int[] sortedEntities = sortedGroupIds.computeIfAbsent( + referenceName, + theReferenceName -> { + final ConstantFormula unsortedIds = new ConstantFormula(groupIdIndex.get(theReferenceName)); + final Bitmap unsortedIdsBitmap = unsortedIds.compute(); + final int[] result = new int[unsortedIdsBitmap.size()]; + final int peak = sorter.sortAndSlice( + unsortedIds, 0, unsortedIdsBitmap.size(), result, 0 + ); + return SortUtils.asResult(result, peak); + } + ); + return Integer.compare( + ArrayUtils.indexOf(o1.getGroupId(), sortedEntities), + ArrayUtils.indexOf(o2.getGroupId(), sortedEntities) + ); + } else { + if (o1.getGroupId() == null) { + return 1; + } else if (o2.getGroupId() == null) { + return -1; + } else { + return Integer.compare(o1.getGroupId(), o2.getGroupId()); + } + } + } + /** * Returns TRUE if facet with `facetId` of specified `referenceName` was requested by the user. */ @@ -656,58 +725,6 @@ public Function, Collection characteristics() { return Set.of(Characteristics.UNORDERED); } - - @Nonnull - private int[] getSortedFacets(Map theFacetStatistics, Sorter sorter) { - // if the sorter is defined, sort them - final RoaringBitmapWriter writer = RoaringBitmapBackedBitmap.buildWriter(); - // collect all entity primary keys - theFacetStatistics.keySet().forEach(writer::add); - // create sorted array using the sorter - final ConstantFormula unsortedIds = new ConstantFormula(new BaseBitmap(writer.get())); - final Bitmap recordsToSort = unsortedIds.compute(); - final int count = recordsToSort.size(); - final int[] result = new int[count]; - final int peak = sorter.sortAndSlice( - queryContext, unsortedIds, 0, count, result, 0 - ); - return SortUtils.asResult(result, peak); - } - - private int compareFacetGroupSummaries(Map groupIdIndex, Map sortedGroupIds, GroupAccumulator o1, GroupAccumulator o2) { - if (o1.getReferenceSchema() != o2.getReferenceSchema()) { - return Integer.compare(o1.getFacetSummaryRequest().order(), o2.getFacetSummaryRequest().order()); - } else if (o1.getFacetSummaryRequest().groupSorter() != null) { - final Sorter sorter = o1.getFacetSummaryRequest().groupSorter(); - // create sorted array using the sorter - final String referenceName = o1.getFacetSummaryRequest().referenceSchema().getName(); - final int[] sortedEntities = sortedGroupIds.computeIfAbsent( - referenceName, - theReferenceName -> { - final ConstantFormula unsortedIds = new ConstantFormula(groupIdIndex.get(theReferenceName)); - final Bitmap unsortedIdsBitmap = unsortedIds.compute(); - final int[] result = new int[unsortedIdsBitmap.size()]; - final int peak = sorter.sortAndSlice( - queryContext, unsortedIds, 0, unsortedIdsBitmap.size(), result, 0 - ); - return SortUtils.asResult(result, peak); - } - ); - return Integer.compare( - ArrayUtils.indexOf(o1.getGroupId(), sortedEntities), - ArrayUtils.indexOf(o2.getGroupId(), sortedEntities) - ); - } else { - if (o1.getGroupId() == null) { - return 1; - } else if (o2.getGroupId() == null) { - return -1; - } else { - return Integer.compare(o1.getGroupId(), o2.getGroupId()); - } - } - } - } /** @@ -917,8 +934,8 @@ private record FacetSummaryRequest( @Nonnull ReferenceSchemaContract referenceSchema, @Nullable IntPredicate facetPredicate, @Nullable IntPredicate groupPredicate, - @Nullable Sorter facetSorter, - @Nullable Sorter groupSorter, + @Nullable NestedContextSorter facetSorter, + @Nullable NestedContextSorter groupSorter, @Nullable EntityFetch facetEntityRequirement, @Nullable EntityGroupFetch groupEntityRequirement, @Nonnull BiFunction facetEntityFetcher, @@ -954,8 +971,8 @@ public Function getGroupEntityFetcher( private record DefaultFacetSummaryRequest( @Nullable Function facetPredicate, @Nullable Function groupPredicate, - @Nullable Function facetSorter, - @Nullable Function groupSorter, + @Nullable Function facetSorter, + @Nullable Function groupSorter, @Nullable EntityFetch facetEntityRequirement, @Nullable EntityGroupFetch groupEntityRequirement, @Nonnull FacetStatisticsDepth facetStatisticsDepth diff --git a/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/hierarchyStatistics/HierarchyOfReferenceTranslator.java b/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/hierarchyStatistics/HierarchyOfReferenceTranslator.java index 382b6e138..bb2fa3bf5 100644 --- a/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/hierarchyStatistics/HierarchyOfReferenceTranslator.java +++ b/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/hierarchyStatistics/HierarchyOfReferenceTranslator.java @@ -34,13 +34,14 @@ import io.evitadb.api.requestResponse.data.mutation.reference.ReferenceKey; import io.evitadb.api.requestResponse.schema.EntitySchemaContract; import io.evitadb.api.requestResponse.schema.ReferenceSchemaContract; +import io.evitadb.core.EntityCollection; import io.evitadb.core.query.algebra.base.EmptyFormula; import io.evitadb.core.query.common.translator.SelfTraversingTranslator; import io.evitadb.core.query.extraResult.ExtraResultPlanningVisitor; import io.evitadb.core.query.extraResult.ExtraResultProducer; import io.evitadb.core.query.extraResult.translator.RequireConstraintTranslator; import io.evitadb.core.query.extraResult.translator.hierarchyStatistics.producer.HierarchyStatisticsProducer; -import io.evitadb.core.query.sort.Sorter; +import io.evitadb.core.query.sort.NestedContextSorter; import io.evitadb.index.EntityIndexKey; import io.evitadb.index.EntityIndexType; import io.evitadb.index.GlobalEntityIndex; @@ -92,15 +93,15 @@ public ExtraResultProducer apply(HierarchyOfReference hierarchyOfReference, Extr () -> new EntityIsNotHierarchicalException(referenceName, entityType)); final HierarchyFilterConstraint hierarchyWithin = evitaRequest.getHierarchyWithin(referenceName); - final Optional targetGlobalIndexRef = extraResultPlanner.getGlobalEntityIndexIfExists(entityType); - if (targetGlobalIndexRef.isPresent()) { - final GlobalEntityIndex globalIndex = targetGlobalIndexRef.get(); - final Sorter sorter = hierarchyOfReference.getOrderBy() + final Optional targetCollectionRef = extraResultPlanner.getEntityCollection(entityType); + final GlobalEntityIndex globalIndex = targetCollectionRef.flatMap(EntityCollection::getGlobalIndexIfExists).orElse(null); + if (globalIndex != null) { + final NestedContextSorter sorter = hierarchyOfReference.getOrderBy() .map( it -> extraResultPlanner.createSorter( it, null, - globalIndex, + targetCollectionRef.get(), entityType, () -> "Hierarchy statistics of `" + entitySchema.getName() + "`: " + it ) diff --git a/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/hierarchyStatistics/HierarchyOfSelfTranslator.java b/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/hierarchyStatistics/HierarchyOfSelfTranslator.java index 78a0d3ec6..acc12ef86 100644 --- a/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/hierarchyStatistics/HierarchyOfSelfTranslator.java +++ b/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/hierarchyStatistics/HierarchyOfSelfTranslator.java @@ -32,6 +32,7 @@ import io.evitadb.api.query.require.StatisticsBase; import io.evitadb.api.requestResponse.EvitaRequest; import io.evitadb.api.requestResponse.schema.EntitySchemaContract; +import io.evitadb.core.EntityCollection; import io.evitadb.core.query.algebra.Formula; import io.evitadb.core.query.algebra.base.ConstantFormula; import io.evitadb.core.query.algebra.utils.FormulaFactory; @@ -40,7 +41,7 @@ import io.evitadb.core.query.extraResult.ExtraResultProducer; import io.evitadb.core.query.extraResult.translator.RequireConstraintTranslator; import io.evitadb.core.query.extraResult.translator.hierarchyStatistics.producer.HierarchyStatisticsProducer; -import io.evitadb.core.query.sort.Sorter; +import io.evitadb.core.query.sort.NestedContextSorter; import io.evitadb.index.GlobalEntityIndex; import io.evitadb.index.bitmap.BaseBitmap; import io.evitadb.index.hierarchy.predicate.FilteringFormulaHierarchyEntityPredicate; @@ -83,15 +84,15 @@ public ExtraResultProducer apply(HierarchyOfSelf hierarchyOfSelf, ExtraResultPla // we need to register producer prematurely extraResultPlanner.registerProducer(hierarchyStatisticsProducer); - final Optional targetGlobalIndexRef = extraResultPlanner.getGlobalEntityIndexIfExists(queriedEntityType); - if (targetGlobalIndexRef.isPresent()) { - final GlobalEntityIndex globalIndex = targetGlobalIndexRef.get(); - final Sorter sorter = hierarchyOfSelf.getOrderBy() + final Optional targetCollectionRef = extraResultPlanner.getEntityCollection(queriedEntityType); + final GlobalEntityIndex globalIndex = targetCollectionRef.flatMap(EntityCollection::getGlobalIndexIfExists).orElse(null); + if (globalIndex != null) { + final NestedContextSorter sorter = hierarchyOfSelf.getOrderBy() .map( it -> extraResultPlanner.createSorter( it, null, - globalIndex, + targetCollectionRef.get(), queriedEntityType, () -> "Hierarchy statistics of `" + entitySchema.getName() + "`: " + it ) diff --git a/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/hierarchyStatistics/producer/HierarchySet.java b/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/hierarchyStatistics/producer/HierarchySet.java index 9735e3b50..9c5220109 100644 --- a/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/hierarchyStatistics/producer/HierarchySet.java +++ b/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/hierarchyStatistics/producer/HierarchySet.java @@ -24,11 +24,10 @@ package io.evitadb.core.query.extraResult.translator.hierarchyStatistics.producer; import io.evitadb.api.requestResponse.extraResult.Hierarchy.LevelInfo; -import io.evitadb.core.query.QueryContext; import io.evitadb.core.query.algebra.Formula; import io.evitadb.core.query.algebra.base.ConstantFormula; import io.evitadb.core.query.algebra.base.EmptyFormula; -import io.evitadb.core.query.sort.Sorter; +import io.evitadb.core.query.sort.NestedContextSorter; import io.evitadb.core.query.sort.utils.SortUtils; import io.evitadb.index.bitmap.BaseBitmap; import io.evitadb.index.bitmap.RoaringBitmapBackedBitmap; @@ -56,10 +55,6 @@ */ @RequiredArgsConstructor public class HierarchySet { - /** - * Reference to the query context that allows to access entity bodies. - */ - private final QueryContext queryContext; /** * The list contains all registered hierarchy computers along with the string key their output will be indexed. */ @@ -69,7 +64,7 @@ public class HierarchySet { * by its primary key in ascending order. */ @Nullable - private Sorter sorter; + private NestedContextSorter sorter; /** * Adds all {@link LevelInfo#entity()} primary keys to the `writer` traversing them recursively so that all entities @@ -114,7 +109,7 @@ private static List sort(@Nonnull List result, @Nonnull in /** * Initializes the {@link #sorter} field. */ - public void setSorter(@Nullable Sorter sorter) { + public void setSorter(@Nullable NestedContextSorter sorter) { this.sorter = sorter; } @@ -150,7 +145,7 @@ public Map> createStatistics(@Nullable Locale language) final Formula levelIdFormula = bitmap.isEmpty() ? EmptyFormula.INSTANCE : new ConstantFormula(new BaseBitmap(bitmap)); final int[] sortedEntities = new int[levelIdFormula.compute().size()]; final int sortedEntitiesPeak = sorter.sortAndSlice( - queryContext, levelIdFormula, 0, levelIdFormula.compute().size(), sortedEntities, 0 + levelIdFormula, 0, levelIdFormula.compute().size(), sortedEntities, 0 ); // replace the output with the sorted one final int[] normalizedSortedResult = SortUtils.asResult(sortedEntities, sortedEntitiesPeak); diff --git a/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/hierarchyStatistics/producer/HierarchyStatisticsProducer.java b/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/hierarchyStatistics/producer/HierarchyStatisticsProducer.java index 25b4c9c3e..1bb3555fc 100644 --- a/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/hierarchyStatistics/producer/HierarchyStatisticsProducer.java +++ b/evita_engine/src/main/java/io/evitadb/core/query/extraResult/translator/hierarchyStatistics/producer/HierarchyStatisticsProducer.java @@ -42,7 +42,7 @@ import io.evitadb.core.query.QueryContext; import io.evitadb.core.query.algebra.Formula; import io.evitadb.core.query.extraResult.ExtraResultProducer; -import io.evitadb.core.query.sort.Sorter; +import io.evitadb.core.query.sort.NestedContextSorter; import io.evitadb.exception.EvitaInvalidUsageException; import io.evitadb.function.IntBiFunction; import io.evitadb.index.GlobalEntityIndex; @@ -153,7 +153,7 @@ public void interpret( @Nonnull IntBiFunction directlyQueriedEntitiesFormulaProducer, @Nullable Function hierarchyFilterPredicateProducer, @Nonnull EmptyHierarchicalEntityBehaviour behaviour, - @Nullable Sorter sorter, + @Nullable NestedContextSorter sorter, @Nonnull Runnable interpretationLambda ) { Assert.isTrue(context.get() == null, "HierarchyOfSelf / HierarchyOfReference cannot be nested inside each other!"); @@ -196,13 +196,13 @@ public void addComputer( final HierarchyProducerContext ctx = getContext(constraintName); if (ctx.referenceSchema() == null) { if (this.selfHierarchyRequest == null) { - this.selfHierarchyRequest = new HierarchySet(queryContext); + this.selfHierarchyRequest = new HierarchySet(); } this.selfHierarchyRequest.addComputer(outputName, computer); } else { this.hierarchyRequests.computeIfAbsent( ctx.referenceSchema().getName(), - s -> new HierarchySet(queryContext) + s -> new HierarchySet() ) .addComputer(outputName, computer); } diff --git a/evita_engine/src/main/java/io/evitadb/core/query/sort/ConditionalSorter.java b/evita_engine/src/main/java/io/evitadb/core/query/sort/ConditionalSorter.java index f78e88a5a..80821ccde 100644 --- a/evita_engine/src/main/java/io/evitadb/core/query/sort/ConditionalSorter.java +++ b/evita_engine/src/main/java/io/evitadb/core/query/sort/ConditionalSorter.java @@ -50,7 +50,7 @@ static Sorter getFirstApplicableSorter(@Nullable Sorter sorter, @Nonnull QueryCo } /** - * Method must return TRUE in case the sorter {@link #sortAndSlice(QueryContext, Formula, int, int)} should be + * Method must return TRUE in case the sorter {@link #sortAndSlice(QueryContext, Formula, int, int, int[], int)} should be * applied on the query result. */ boolean shouldApply(@Nonnull QueryContext queryContext); diff --git a/evita_engine/src/main/java/io/evitadb/core/query/sort/NestedContextSorter.java b/evita_engine/src/main/java/io/evitadb/core/query/sort/NestedContextSorter.java new file mode 100644 index 000000000..8ff63be51 --- /dev/null +++ b/evita_engine/src/main/java/io/evitadb/core/query/sort/NestedContextSorter.java @@ -0,0 +1,58 @@ +/* + * + * _ _ ____ ____ + * _____ _(_) |_ __ _| _ \| __ ) + * / _ \ \ / / | __/ _` | | | | _ \ + * | __/\ V /| | || (_| | |_| | |_) | + * \___| \_/ |_|\__\__,_|____/|____/ + * + * Copyright (c) 2024 + * + * Licensed under the Business Source License, Version 1.1 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://github.com/FgForrest/evitaDB/blob/main/LICENSE + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.evitadb.core.query.sort; + +import io.evitadb.core.query.QueryContext; +import io.evitadb.core.query.algebra.AbstractFormula; +import io.evitadb.core.query.algebra.Formula; +import lombok.RequiredArgsConstructor; + +import javax.annotation.Nonnull; + +/** + * This class is a wrapper for {@link Sorter} along with correct nested {@link QueryContext} initialized for proper + * query and entity type. + * + * @author Jan Novotný (novotny@fg.cz), FG Forrest a.s. (c) 2024 + */ +@RequiredArgsConstructor +public class NestedContextSorter { + private final QueryContext context; + private final Sorter sorter; + + /** + * Method sorts output of the {@link AbstractFormula} input and extracts slice of the result data between `startIndex` (inclusive) + * and `endIndex` (exclusive). + */ + public int sortAndSlice( + @Nonnull Formula input, + int startIndex, + int endIndex, + @Nonnull int[] result, + int peak + ) { + return sorter.sortAndSlice(context, input, startIndex, endIndex, result, peak); + } + +} diff --git a/evita_external_api/evita_external_api_core/src/main/java/io/evitadb/externalApi/http/ExternalApiExceptionHandler.java b/evita_external_api/evita_external_api_core/src/main/java/io/evitadb/externalApi/http/ExternalApiExceptionHandler.java index c7cf09228..1b8ff27e8 100644 --- a/evita_external_api/evita_external_api_core/src/main/java/io/evitadb/externalApi/http/ExternalApiExceptionHandler.java +++ b/evita_external_api/evita_external_api_core/src/main/java/io/evitadb/externalApi/http/ExternalApiExceptionHandler.java @@ -72,9 +72,7 @@ public void handleRequest(@Nonnull HttpServerExchange exchange) throws Exception if (evitaError instanceof final ExternalApiInternalError externalApiInternalError) { // log any API internal errors that Evita cannot handle because they are outside of Evita execution log.error( - "Internal Evita " + getExternalApiCode() + " API error occurred in {}: {}", - externalApiInternalError.getErrorCode(), - externalApiInternalError.getPrivateMessage(), + "Internal Evita " + getExternalApiCode() + " API error occurred in " + externalApiInternalError.getErrorCode() + ": " + externalApiInternalError.getPrivateMessage(), externalApiInternalError ); }