Implement construct PrefilterExpression from SparqlExpression.

src/engine/Filter.cpp

@@ -27,7 +27,9 @@ Filter::Filter(QueryExecutionContext* qec,
                sparqlExpression::SparqlExpressionPimpl expression)
     : Operation(qec),
       _subtree(std::move(subtree)),
-      _expression{std::move(expression)} {}
+      _expression{std::move(expression)} {
+  setPrefilterExpressionForIndexScanChildren();
+}
 
 // _____________________________________________________________________________
 string Filter::getCacheKeyImpl() const {
@@ -37,10 +39,22 @@ string Filter::getCacheKeyImpl() const {
   return std::move(os).str();
 }
 
+//______________________________________________________________________________
 string Filter::getDescriptor() const {
   return absl::StrCat("Filter ", _expression.getDescriptor());
 }
 
+//______________________________________________________________________________
+void Filter::setPrefilterExpressionForIndexScanChildren() {
+  const std::vector<PrefilterVariablePair>& prefilterVec =
+      _expression.getPrefilterExpressionForMetadata();
+  this->forAllDescendants([&prefilterVec](const QueryExecutionTree* ptr) {
+    if (ptr) {
+      ptr->setPrefilterExpression(prefilterVec);
+    }
+  });
+}
+
 // _____________________________________________________________________________
 ProtoResult Filter::computeResult(bool requestLaziness) {
   LOG(DEBUG) << "Getting sub-result for Filter result computation..." << endl;

src/engine/Filter.h

@@ -10,10 +10,11 @@
 
 #include "engine/Operation.h"
 #include "engine/QueryExecutionTree.h"
-#include "engine/sparqlExpressions/SparqlExpressionPimpl.h"
 #include "parser/ParsedQuery.h"
 
 class Filter : public Operation {
+  using PrefilterVariablePair = sparqlExpression::PrefilterExprVariablePair;
+
  private:
   std::shared_ptr<QueryExecutionTree> _subtree;
   sparqlExpression::SparqlExpressionPimpl _expression;
@@ -58,6 +59,8 @@ class Filter : public Operation {
     return _subtree->getVariableColumns();
   }
 
+  void setPrefilterExpressionForIndexScanChildren();
+
   ProtoResult computeResult(bool requestLaziness) override;
 
   // Perform the actual filter operation of the data provided.

src/engine/IndexScan.cpp

@@ -17,13 +17,15 @@
 
 // _____________________________________________________________________________
 IndexScan::IndexScan(QueryExecutionContext* qec, Permutation::Enum permutation,
-                     const SparqlTripleSimple& triple, Graphs graphsToFilter)
+                     const SparqlTripleSimple& triple, Graphs graphsToFilter,
+                     PrefilterIndexPairs prefilters)
     : Operation(qec),
       permutation_(permutation),
       subject_(triple.s_),
       predicate_(triple.p_),
       object_(triple.o_),
       graphsToFilter_{std::move(graphsToFilter)},
+      prefilters_{std::move(prefilters)},
       numVariables_(static_cast<size_t>(subject_.isVariable()) +
                     static_cast<size_t>(predicate_.isVariable()) +
                     static_cast<size_t>(object_.isVariable())) {
@@ -51,9 +53,10 @@
 
 // _____________________________________________________________________________
 IndexScan::IndexScan(QueryExecutionContext* qec, Permutation::Enum permutation,
-                     const SparqlTriple& triple, Graphs graphsToFilter)
-    : IndexScan(qec, permutation, triple.getSimple(),
-                std::move(graphsToFilter)) {}
+                     const SparqlTriple& triple, Graphs graphsToFilter,
+                     PrefilterIndexPairs prefilters)
+    : IndexScan(qec, permutation, triple.getSimple(), std::move(graphsToFilter),
+                std::move(prefilters)) {}
 
 // _____________________________________________________________________________
 string IndexScan::getCacheKeyImpl() const {
@@ -115,6 +118,24 @@
   return result;
 }
 
+// _____________________________________________________________________________
+void IndexScan::setPrefilterExpression(
+    const std::vector<PrefilterVariablePair>& prefilterVariablePairs) {
+  const std::vector<ColumnIndex>& sortedColumns = resultSortedOn();
+  VariableToColumnMap varToColMap = computeVariableToColumnMap();
+
+  const auto addPrefilterIfSorted = [&](const PrefilterVariablePair& pair) {
+    const Variable& variable = pair.second;
+    if (varToColMap.find(variable) != varToColMap.end()) {
+      const ColumnIndex colIdx = varToColMap[variable].columnIndex_;
+      if (std::ranges::find(sortedColumns, colIdx) != sortedColumns.end()) {
+        prefilters_.push_back(std::make_pair(pair.first->clone(), colIdx));
+      }
+    }
+  };
+  std::ranges::for_each(prefilterVariablePairs, addPrefilterIfSorted);
+}
+
 // _____________________________________________________________________________
 VariableToColumnMap IndexScan::computeVariableToColumnMap() const {
   VariableToColumnMap variableToColumnMap;
@@ -134,40 +155,63 @@
   return variableToColumnMap;
 }
 
+//______________________________________________________________________________
+std::vector<CompressedBlockMetadata> IndexScan::applyFilterBlockMetadata(
+    std::vector<CompressedBlockMetadata>&& blocks) const {
+  std::ranges::for_each(prefilters_, [&blocks](const PrefilterIndexPair& pair) {
+    pair.first->evaluate(blocks, pair.second);
+  });
+  return blocks;
+};
+
 // _____________________________________________________________________________
-Result::Generator IndexScan::scanInChunks() const {
+Result::Generator IndexScan::chunkedIndexScan() const {
   auto metadata = getMetadataForScan();
   if (!metadata.has_value()) {
     co_return;
   }
   auto blocksSpan =
       CompressedRelationReader::getBlocksFromMetadata(metadata.value());
-  std::vector<CompressedBlockMetadata> blocks{blocksSpan.begin(),
-                                              blocksSpan.end()};
-  for (IdTable& idTable : getLazyScan(std::move(blocks))) {
+  for (IdTable& idTable : getLazyScan({blocksSpan.begin(), blocksSpan.end()})) {
     co_yield {std::move(idTable), LocalVocab{}};
   }
 }
 
 // _____________________________________________________________________________
-ProtoResult IndexScan::computeResult(bool requestLaziness) {
-  LOG(DEBUG) << "IndexScan result computation...\n";
-  if (requestLaziness) {
-    return {scanInChunks(), resultSortedOn()};
-  }
-  IdTable idTable{getExecutionContext()->getAllocator()};
-
+IdTable IndexScan::completeIndexScan() const {
+  // Get the blocks.
+  auto metadata = getMetadataForScan();
+  auto blockSpan =
+      metadata.has_value()
+          ? CompressedRelationReader::getBlocksFromMetadata(metadata.value())
+          : std::span<const CompressedBlockMetadata>{};
+  auto optFilteredBlocks = getLimit().isUnconstrained()
+                               ? std::optional{applyFilterBlockMetadata(
+                                     {blockSpan.begin(), blockSpan.end()})}
+                               : std::nullopt;
+  // Create IdTable, fill it with content by performing scan().
   using enum Permutation::Enum;
+  IdTable idTable{getExecutionContext()->getAllocator()};
   idTable.setNumColumns(numVariables_);
   const auto& index = _executionContext->getIndex();
-  idTable =
-      index.scan(getScanSpecification(), permutation_, additionalColumns(),
-                 cancellationHandle_, locatedTriplesSnapshot(), getLimit());
+  idTable = index.getImpl()
+                .getPermutation(permutation_)
+                .scan(getScanSpecification(), additionalColumns(),
+                      cancellationHandle_, locatedTriplesSnapshot(), getLimit(),
+                      optFilteredBlocks);
   AD_CORRECTNESS_CHECK(idTable.numColumns() == getResultWidth());
   LOG(DEBUG) << "IndexScan result computation done.\n";
   checkCancellation();
+  return idTable;
+}
 
-  return {std::move(idTable), resultSortedOn(), LocalVocab{}};
+// _____________________________________________________________________________
+ProtoResult IndexScan::computeResult(bool requestLaziness) {
+  LOG(DEBUG) << "IndexScan result computation...\n";
+  if (requestLaziness) {
+    return {chunkedIndexScan(), resultSortedOn()};
+  }
+  return {completeIndexScan(), getResultSortedOn(), LocalVocab{}};
 }
 
 // _____________________________________________________________________________
@@ -208,7 +252,7 @@
   AD_CONTRACT_CHECK(multiplicity_.size() == getResultWidth());
 }
 
-// ___________________________________________________________________________
+// _____________________________________________________________________________
 std::array<const TripleComponent* const, 3> IndexScan::getPermutedTriple()
     const {
   std::array triple{&subject_, &predicate_, &object_};
@@ -217,30 +261,30 @@
           triple[permutation[2]]};
 }
 
-// ___________________________________________________________________________
+// _____________________________________________________________________________
 ScanSpecification IndexScan::getScanSpecification() const {
   const IndexImpl& index = getIndex().getImpl();
   return getScanSpecificationTc().toScanSpecification(index);
 }
 
-// ___________________________________________________________________________
+// _____________________________________________________________________________
 ScanSpecificationAsTripleComponent IndexScan::getScanSpecificationTc() const {
   auto permutedTriple = getPermutedTriple();
   return {*permutedTriple[0], *permutedTriple[1], *permutedTriple[2],
           graphsToFilter_};
 }
 
-// ___________________________________________________________________________
+// _____________________________________________________________________________
 Permutation::IdTableGenerator IndexScan::getLazyScan(
-    std::vector<CompressedBlockMetadata> blocks) const {
+    std::vector<CompressedBlockMetadata>&& blocks) const {
   // If there is a LIMIT or OFFSET clause that constrains the scan
   // (which can happen with an explicit subquery), we cannot use the prefiltered
   // blocks, as we currently have no mechanism to include limits and offsets
   // into the prefiltering (`std::nullopt` means `scan all blocks`).
-  auto actualBlocks = getLimit().isUnconstrained()
-                          ? std::optional{std::move(blocks)}
-                          : std::nullopt;
-
+  auto actualBlocks =
+      getLimit().isUnconstrained()
+          ? std::optional{applyFilterBlockMetadata(std::move(blocks))}
+          : std::nullopt;
   return getIndex()
       .getImpl()
       .getPermutation(permutation())
@@ -249,15 +293,15 @@
                 locatedTriplesSnapshot(), getLimit());
 };
 
-// ________________________________________________________________
+// _____________________________________________________________________________
 std::optional<Permutation::MetadataAndBlocks> IndexScan::getMetadataForScan()
     const {
   const auto& index = getExecutionContext()->getIndex().getImpl();
   return index.getPermutation(permutation())
       .getMetadataAndBlocks(getScanSpecification(), locatedTriplesSnapshot());
 };
 
-// ________________________________________________________________
+// _____________________________________________________________________________
 std::array<Permutation::IdTableGenerator, 2>
 IndexScan::lazyScanForJoinOfTwoScans(const IndexScan& s1, const IndexScan& s2) {
   AD_CONTRACT_CHECK(s1.numVariables_ <= 3 && s2.numVariables_ <= 3);
@@ -300,13 +344,14 @@
   auto [blocks1, blocks2] = CompressedRelationReader::getBlocksForJoin(
       metaBlocks1.value(), metaBlocks2.value());
 
-  std::array result{s1.getLazyScan(blocks1), s2.getLazyScan(blocks2)};
+  std::array result{s1.getLazyScan(std::move(blocks1)),
+                    s2.getLazyScan(std::move(blocks2))};
   result[0].details().numBlocksAll_ = metaBlocks1.value().blockMetadata_.size();
   result[1].details().numBlocksAll_ = metaBlocks2.value().blockMetadata_.size();
   return result;
 }
 
-// ________________________________________________________________
+// _____________________________________________________________________________
 Permutation::IdTableGenerator IndexScan::lazyScanForJoinOfColumnWithScan(
     std::span<const Id> joinColumn) const {
   AD_EXPENSIVE_CHECK(std::ranges::is_sorted(joinColumn));
@@ -321,7 +366,7 @@
   auto blocks = CompressedRelationReader::getBlocksForJoin(joinColumn,
                                                            metaBlocks1.value());
 
-  auto result = getLazyScan(blocks);
+  auto result = getLazyScan(std::move(blocks));
   result.details().numBlocksAll_ = metaBlocks1.value().blockMetadata_.size();
   return result;
 }

src/engine/IndexScan.h

@@ -11,13 +11,21 @@ class SparqlTriple;
 class SparqlTripleSimple;
 
 class IndexScan final : public Operation {
+  using Graphs = ScanSpecificationAsTripleComponent::Graphs;
+  // Pair containing a `PrefilterExpression` with `ColumnIndex` (eval. index)
+  using PrefilterIndexPair =
+      std::pair<std::unique_ptr<prefilterExpressions::PrefilterExpression>,
+                ColumnIndex>;
+  // Vector with `PrefilterIndexPair` values.
+  using PrefilterIndexPairs = std::vector<PrefilterIndexPair>;
+
  private:
   Permutation::Enum permutation_;
   TripleComponent subject_;
   TripleComponent predicate_;
   TripleComponent object_;
-  using Graphs = ScanSpecificationAsTripleComponent::Graphs;
   Graphs graphsToFilter_;
+  PrefilterIndexPairs prefilters_;
   size_t numVariables_;
   size_t sizeEstimate_;
   vector<float> multiplicity_;
@@ -30,10 +38,12 @@ class IndexScan final : public Operation {
 
  public:
   IndexScan(QueryExecutionContext* qec, Permutation::Enum permutation,
-            const SparqlTriple& triple, Graphs graphsToFilter = std::nullopt);
+            const SparqlTriple& triple, Graphs graphsToFilter = std::nullopt,
+            PrefilterIndexPairs prefilters = {});
   IndexScan(QueryExecutionContext* qec, Permutation::Enum permutation,
             const SparqlTripleSimple& triple,
-            Graphs graphsToFilter = std::nullopt);
+            Graphs graphsToFilter = std::nullopt,
+            PrefilterIndexPairs prefilters = {});
 
   ~IndexScan() override = default;
 
@@ -55,6 +65,10 @@ class IndexScan final : public Operation {
 
   vector<ColumnIndex> resultSortedOn() const override;
 
+  // Set `PrefilterExpression`s.
+  void setPrefilterExpression(const std::vector<PrefilterVariablePair>&
+                                  prefilterVariablePairs) override;
+
   size_t numVariables() const { return numVariables_; }
 
   // Return the exact result size of the index scan. This is always known as it
@@ -130,10 +144,19 @@ class IndexScan final : public Operation {
 
   VariableToColumnMap computeVariableToColumnMap() const override;
 
-  Result::Generator scanInChunks() const;
+  // Filter relevant `CompressedBlockMetadata` blocks by applying the
+  // `PrefilterExpression`s from `prefilters_`.
+  std::vector<CompressedBlockMetadata> applyFilterBlockMetadata(
+      std::vector<CompressedBlockMetadata>&& blocks) const;
+
+  // Return the (lazy) `IdTable` for this `IndexScan` in chunks.
+  Result::Generator chunkedIndexScan() const;
+  // Get the `IdTable` for this `IndexScan` in one piece.
+  IdTable completeIndexScan() const;
 
-  //  Helper functions for the public `getLazyScanFor...` functions (see above).
+  // Helper functions for the public `getLazyScanFor...` methods and
+  // `chunkedIndexScan` (see above).
   Permutation::IdTableGenerator getLazyScan(
-      std::vector<CompressedBlockMetadata> blocks) const;
+      std::vector<CompressedBlockMetadata>&& blocks) const;
   std::optional<Permutation::MetadataAndBlocks> getMetadataForScan() const;
 };