refactor: reflected references are not allowed across scopes

Reflected references were allowed originally only in the very same scope. In reality we need those across scopes - if the reference schemas are indexed in both of scopes.

This allows us to look up for entities by reference having / entity having in multiple scopes at once.

evita_engine/src/main/java/io/evitadb/core/query/filter/FilterByVisitor.java

@@ -846,7 +846,33 @@ public Formula getReferencedRecordIdFormula(
 						(theEntity, attributeName, locale) -> theEntity.getReferences(referenceName).stream().map(it -> it.getAttributeValue(attributeName, locale)),
 						() -> {
 							filterBy.accept(this);
-							return getFormulaAndClear();
+							final Formula formula = getFormulaAndClear();
+							// when target entity type is managed
+							if (referenceSchema.isReferencedEntityTypeManaged()) {
+								// we must match the result with the existence of the primary keys in the global entity index in allowed scopes
+								return FormulaFactory.and(
+									formula,
+									FormulaFactory.or(
+										// here we use all scopes targeted by the query
+										this.queryContext.getScopes()
+											.stream()
+											.map(theScope -> {
+												if (referenceSchema.isIndexedInScope(theScope)) {
+													return getGlobalEntityIndexIfExists(referenceSchema.getReferencedEntityType(), theScope)
+														.map(EntityIndex::getAllPrimaryKeysFormula)
+														.orElse(EmptyFormula.INSTANCE);
+												} else {
+													// if the schema is not indexed in particular scope, we must keep original formula results in place
+													// to avoid their removal from the final result
+													return formula;
+												}
+											})
+											.toArray(Formula[]::new)
+									)
+								);
+							} else {
+								return formula;
+							}
 						}
 					);
 				})

evita_engine/src/main/java/io/evitadb/core/query/filter/translator/reference/ReferenceHavingTranslator.java

@@ -61,44 +61,29 @@
  */
 public class ReferenceHavingTranslator implements FilteringConstraintTranslator<ReferenceHaving>, SelfTraversingTranslator {
 
-	@Nonnull
-	@Override
-	public Formula translate(@Nonnull ReferenceHaving referenceHaving, @Nonnull FilterByVisitor filterByVisitor) {
-		final String referenceName = referenceHaving.getReferenceName();
-		final ProcessingScope<? extends Index<?>> processingScope = filterByVisitor.getProcessingScope();
-		final EntitySchemaContract entitySchema = processingScope.getEntitySchema();
-
-		Assert.isTrue(
-			entitySchema != null,
-			() -> "Entity type must be known when filtering by `referenceHaving`."
-		);
-
-		final ReferenceSchemaContract referenceSchema = entitySchema.getReference(referenceName)
-			.orElseThrow(() -> new ReferenceNotFoundException(referenceName, entitySchema));
-
-		final Supplier<List<ReducedEntityIndex>> referencedEntityIndexesSupplier = () -> getTargetIndexes(
-			filterByVisitor, referenceHaving, processingScope.getScopes()
-		);
-
-		// the reference content needs to be prefetched in order to bea able to apply the filter on prefetched data
-		// i.e. access the reference attributes
-		filterByVisitor.addRequirementToPrefetch(referenceContent(referenceName));
-
-		return applySearchOnIndexes(
-			referenceHaving, filterByVisitor, entitySchema, referenceSchema, referencedEntityIndexesSupplier
-		);
-	}
-
+	/**
+	 * Applies a search operation on specified indexes based on the given filter constraints, context, and schema
+	 * configurations. The method builds and executes the necessary formulas for filtering and efficiently retrieves
+	 * the data satisfying the filter criteria.
+	 *
+	 * @param filterConstraint              the filtering constraint specifying conditions to evaluate against the references.
+	 * @param filterByVisitor               the visitor responsible for collecting the results of the filtering logic across schemas.
+	 * @param entitySchema                  the entity schema defining the structure and attributes of the entity being queried.
+	 * @param referenceSchema               the reference schema containing metadata about the reference relation and its attributes.
+	 * @param processingScope               the processing scope providing additional contextual properties required during the filtering.
+	 * @param referencedEntityIndexSupplier the supplier that provides a list of reduced entity indexes for processing references.
+	 * @return the resulting formula representing the combined computation steps for the filtering operation.
+	 */
 	@Nonnull
 	private static Formula applySearchOnIndexes(
 		@Nonnull ReferenceHaving filterConstraint,
 		@Nonnull FilterByVisitor filterByVisitor,
 		@Nonnull EntitySchemaContract entitySchema,
 		@Nonnull ReferenceSchemaContract referenceSchema,
+		@Nonnull ProcessingScope<?> processingScope,
 		@Nonnull Supplier<List<ReducedEntityIndex>> referencedEntityIndexSupplier
 	) {
 		final String referenceName = referenceSchema.getName();
-		final ProcessingScope<?> processingScope = filterByVisitor.getProcessingScope();
 		return filterByVisitor.executeInContextAndIsolatedFormulaStack(
 			ReducedEntityIndex.class,
 			referencedEntityIndexSupplier,
@@ -118,6 +103,8 @@ private static Formula applySearchOnIndexes(
 				getFilterByFormula(filterConstraint).ifPresent(it -> it.accept(filterByVisitor));
 				final Formula[] collectedFormulas = filterByVisitor.getCollectedFormulasOnCurrentLevel();
 				return switch (collectedFormulas.length) {
+					// when there was no filter constraint or entityPrimaryKeyInSet, we can safely use super set formula
+					// e.g. all primary keys in reduced entity indexes
 					case 0 -> filterByVisitor.getSuperSetFormula();
 					case 1 -> collectedFormulas[0];
 					default -> new OrFormula(collectedFormulas);
@@ -144,7 +131,7 @@ private static List<ReducedEntityIndex> getTargetIndexes(
 		@Nonnull FilterByVisitor filterByVisitor,
 		@Nonnull ReferenceHaving referenceHaving,
 		@Nonnull Set<Scope> scopes
-		) {
+	) {
 		final TargetIndexes<?> targetIndexes = filterByVisitor.findTargetIndexSet(referenceHaving);
 		final List<ReducedEntityIndex> referencedEntityIndexes;
 		if (targetIndexes == null) {
@@ -156,4 +143,33 @@ private static List<ReducedEntityIndex> getTargetIndexes(
 		return referencedEntityIndexes;
 	}
 
+	@Nonnull
+	@Override
+	public Formula translate(@Nonnull ReferenceHaving referenceHaving, @Nonnull FilterByVisitor filterByVisitor) {
+		final String referenceName = referenceHaving.getReferenceName();
+		final ProcessingScope<? extends Index<?>> processingScope = filterByVisitor.getProcessingScope();
+		final EntitySchemaContract entitySchema = processingScope.getEntitySchema();
+
+		Assert.isTrue(
+			entitySchema != null,
+			() -> "Entity type must be known when filtering by `referenceHaving`."
+		);
+
+		final ReferenceSchemaContract referenceSchema = entitySchema.getReference(referenceName)
+			.orElseThrow(() -> new ReferenceNotFoundException(referenceName, entitySchema));
+
+		final Supplier<List<ReducedEntityIndex>> referencedEntityIndexesSupplier = () -> getTargetIndexes(
+			filterByVisitor, referenceHaving, processingScope.getScopes()
+		);
+
+		// the reference content needs to be prefetched in order to bea able to apply the filter on prefetched data
+		// i.e. access the reference attributes
+		filterByVisitor.addRequirementToPrefetch(referenceContent(referenceName));
+
+		return applySearchOnIndexes(
+			referenceHaving, filterByVisitor, entitySchema, referenceSchema,
+			processingScope, referencedEntityIndexesSupplier
+		);
+	}
+
 }

evita_engine/src/main/java/io/evitadb/core/query/indexSelection/IndexSelectionVisitor.java

@@ -236,16 +236,21 @@ private void addReferenceIndexOption(@Nonnull ReferenceHaving constraint) {
 		final List<ReducedEntityIndex> theTargetIndexes = theFilterByVisitor
 			.getReferencedRecordEntityIndexes(constraint, scopes);
 
-		// add indexes as potential target indexes
-		this.targetIndexes.add(
-			new TargetIndexes<>(
-				EntityIndexType.REFERENCED_ENTITY.name() +
-					" composed of " + theTargetIndexes.size() + " indexes",
-				constraint,
-				ReducedEntityIndex.class,
-				theTargetIndexes
-			)
-		);
+		if (theTargetIndexes.isEmpty() && !scopes.equals(theFilterByVisitor.getScopes())) {
+			// if the scopes were redefined in processing scope (differ from globally allowed scopes)
+			// skip this indexing option
+		} else {
+			// add indexes as potential target indexes
+			this.targetIndexes.add(
+				new TargetIndexes<>(
+					EntityIndexType.REFERENCED_ENTITY.name() +
+						" composed of " + theTargetIndexes.size() + " indexes",
+					constraint,
+					ReducedEntityIndex.class,
+					theTargetIndexes
+				)
+			);
+		}
 	}
 
 	private FilterByVisitor getFilterByVisitor() {