Merge ssh://github.com/ad-freiburg/qlever into service-spatial-search

ad-freiburg · Nov 29, 2024 · 521f81f · 521f81f
2 parents 954af1e + 527f7bf
commit 521f81f
Show file tree

Hide file tree

Showing 11 changed files with 173 additions and 67 deletions.
diff --git a/src/engine/QueryPlanner.cpp b/src/engine/QueryPlanner.cpp
@@ -1203,13 +1203,6 @@ void QueryPlanner::applyFiltersIfPossible(
   // in one go. Changing `row` inside the loop would invalidate the iterators.
   std::vector<SubtreePlan> addedPlans;
   for (auto& plan : row) {
-    if constexpr (!replace) {
-      if (plan._qet->getRootOperation()->isIndexScanWithNumVariables(3)) {
-        // Do not apply filters to dummies, except at the very end of query
-        // planning.
-        continue;
-      }
-    }
     for (size_t i = 0; i < filters.size(); ++i) {
       if (((plan._idsOfIncludedFilters >> i) & 1) != 0) {
         continue;
@@ -1882,12 +1875,6 @@ std::vector<QueryPlanner::SubtreePlan> QueryPlanner::createJoinCandidates(
 
   // CASE: JOIN ON ONE COLUMN ONLY.
 
-  // Skip if we have two operations, where all three positions are variables.
-  if (a._qet->getRootOperation()->isIndexScanWithNumVariables(3) &&
-      b._qet->getRootOperation()->isIndexScanWithNumVariables(3)) {
-    return candidates;
-  }
-
   // Check if one of the two operations is a HAS_PREDICATE_SCAN.
   // If the join column corresponds to the has-predicate scan's
   // subject column we can use a specialized join that avoids

diff --git a/src/index/DeltaTriples.cpp b/src/index/DeltaTriples.cpp
@@ -219,3 +219,12 @@ void DeltaTriplesManager::clear() { modify(&DeltaTriples::clear); }
 SharedLocatedTriplesSnapshot DeltaTriplesManager::getCurrentSnapshot() const {
   return *currentLocatedTriplesSnapshot_.rlock();
 }
+
+// _____________________________________________________________________________
+void DeltaTriples::setOriginalMetadata(
+    Permutation::Enum permutation,
+    std::vector<CompressedBlockMetadata> metadata) {
+  locatedTriples()
+      .at(static_cast<size_t>(permutation))
+      .setOriginalMetadata(std::move(metadata));
+}
diff --git a/src/index/DeltaTriples.h b/src/index/DeltaTriples.h
@@ -142,6 +142,11 @@ class DeltaTriples {
   // `DeltaTriples` object.
   SharedLocatedTriplesSnapshot getSnapshot() const;
 
+  // Register the original `metadata` for the given `permutation`. This has to
+  // be called before any updates are processed.
+  void setOriginalMetadata(Permutation::Enum permutation,
+                           std::vector<CompressedBlockMetadata> metadata);
+
  private:
   // Find the position of the given triple in the given permutation and add it
   // to each of the six `LocatedTriplesPerBlock` maps (one per permutation).

diff --git a/src/index/IndexImpl.cpp b/src/index/IndexImpl.cpp
@@ -322,7 +322,7 @@ void IndexImpl::updateInputFileSpecificationsAndLog(
   // For a single input stream, show the name and whether we parse in parallel.
   // For multiple input streams, only show the number of streams.
   if (spec.size() == 1) {
-    AD_LOG_INFO << "Parsing triples from single input stream "
+    AD_LOG_INFO << "Processing triples from single input stream "
                 << spec.at(0).filename_ << " (parallel = "
                 << (spec.at(0).parseInParallel_ ? "true" : "false") << ") ..."
                 << std::endl;
@@ -882,14 +882,33 @@ void IndexImpl::createFromOnDiskIndex(const string& onDiskBase) {
            range2.contain(id.getVocabIndex());
   };
 
-  pso_.loadFromDisk(onDiskBase_, isInternalId, true);
-  pos_.loadFromDisk(onDiskBase_, isInternalId, true);
+  // Load the permutations and register the original metadata for the delta
+  // triples.
+  // TODO<joka921> We could delegate the setting of the metadata to the
+  // `Permutation`class, but we first have to deal with The delta triples for
+  // the additional permutations.
+  auto setMetadata = [this](const Permutation& p) {
+    deltaTriplesManager().modify([&p](DeltaTriples& deltaTriples) {
+      deltaTriples.setOriginalMetadata(p.permutation(),
+                                       p.metaData().blockData());
+    });
+  };
+
+  auto load = [this, &isInternalId, &setMetadata](
+                  Permutation& permutation,
+                  bool loadInternalPermutation = false) {
+    permutation.loadFromDisk(onDiskBase_, isInternalId,
+                             loadInternalPermutation);
+    setMetadata(permutation);
+  };
 
+  load(pso_, true);
+  load(pos_, true);
   if (loadAllPermutations_) {
-    ops_.loadFromDisk(onDiskBase_, isInternalId);
-    osp_.loadFromDisk(onDiskBase_, isInternalId);
-    spo_.loadFromDisk(onDiskBase_, isInternalId);
-    sop_.loadFromDisk(onDiskBase_, isInternalId);
+    load(ops_);
+    load(osp_);
+    load(spo_);
+    load(sop_);
   } else {
     AD_LOG_INFO
         << "Only the PSO and POS permutation were loaded, SPARQL queries "

diff --git a/src/index/Permutation.cpp b/src/index/Permutation.cpp
@@ -28,6 +28,7 @@ void Permutation::loadFromDisk(const std::string& onDiskBase,
     internalPermutation_->loadFromDisk(
         absl::StrCat(onDiskBase, QLEVER_INTERNAL_INDEX_INFIX), isInternalId_,
         false);
+    internalPermutation_->isInternalPermutation_ = true;
   }
   if constexpr (MetaData::isMmapBased_) {
     meta_.setup(onDiskBase + ".index" + fileSuffix_ + MMAP_FILE_SUFFIX,
@@ -65,8 +66,9 @@ IdTable Permutation::scan(const ScanSpecification& scanSpec,
   const auto& p = getActualPermutation(scanSpec);
 
   return p.reader().scan(
-      scanSpec, p.meta_.blockData(), additionalColumns, cancellationHandle,
-      getLocatedTriplesForPermutation(locatedTriplesSnapshot), limitOffset);
+      scanSpec, p.getAugmentedMetadataForPermutation(locatedTriplesSnapshot),
+      additionalColumns, cancellationHandle,
+      p.getLocatedTriplesForPermutation(locatedTriplesSnapshot), limitOffset);
 }
 
 // _____________________________________________________________________
@@ -75,8 +77,8 @@ size_t Permutation::getResultSizeOfScan(
     const LocatedTriplesSnapshot& locatedTriplesSnapshot) const {
   const auto& p = getActualPermutation(scanSpec);
   return p.reader().getResultSizeOfScan(
-      scanSpec, p.meta_.blockData(),
-      getLocatedTriplesForPermutation(locatedTriplesSnapshot));
+      scanSpec, p.getAugmentedMetadataForPermutation(locatedTriplesSnapshot),
+      p.getLocatedTriplesForPermutation(locatedTriplesSnapshot));
 }
 
 // _____________________________________________________________________
@@ -85,8 +87,8 @@ std::pair<size_t, size_t> Permutation::getSizeEstimateForScan(
     const LocatedTriplesSnapshot& locatedTriplesSnapshot) const {
   const auto& p = getActualPermutation(scanSpec);
   return p.reader().getSizeEstimateForScan(
-      scanSpec, p.meta_.blockData(),
-      getLocatedTriplesForPermutation(locatedTriplesSnapshot));
+      scanSpec, p.getAugmentedMetadataForPermutation(locatedTriplesSnapshot),
+      p.getLocatedTriplesForPermutation(locatedTriplesSnapshot));
 }
 
 // ____________________________________________________________________________
@@ -95,16 +97,18 @@ IdTable Permutation::getDistinctCol1IdsAndCounts(
     const LocatedTriplesSnapshot& locatedTriplesSnapshot) const {
   const auto& p = getActualPermutation(col0Id);
   return p.reader().getDistinctCol1IdsAndCounts(
-      col0Id, p.meta_.blockData(), cancellationHandle,
-      getLocatedTriplesForPermutation(locatedTriplesSnapshot));
+      col0Id, p.getAugmentedMetadataForPermutation(locatedTriplesSnapshot),
+      cancellationHandle,
+      p.getLocatedTriplesForPermutation(locatedTriplesSnapshot));
 }
 
 // ____________________________________________________________________________
 IdTable Permutation::getDistinctCol0IdsAndCounts(
     const CancellationHandle& cancellationHandle,
     const LocatedTriplesSnapshot& locatedTriplesSnapshot) const {
   return reader().getDistinctCol0IdsAndCounts(
-      meta_.blockData(), cancellationHandle,
+      getAugmentedMetadataForPermutation(locatedTriplesSnapshot),
+      cancellationHandle,
       getLocatedTriplesForPermutation(locatedTriplesSnapshot));
 }
 
@@ -156,8 +160,8 @@ std::optional<CompressedRelationMetadata> Permutation::getMetadata(
     return p.meta_.getMetaData(col0Id);
   }
   return p.reader().getMetadataForSmallRelation(
-      p.meta_.blockData(), col0Id,
-      getLocatedTriplesForPermutation(locatedTriplesSnapshot));
+      p.getAugmentedMetadataForPermutation(locatedTriplesSnapshot), col0Id,
+      p.getLocatedTriplesForPermutation(locatedTriplesSnapshot));
 }
 
 // _____________________________________________________________________
@@ -167,10 +171,11 @@ std::optional<Permutation::MetadataAndBlocks> Permutation::getMetadataAndBlocks(
   const auto& p = getActualPermutation(scanSpec);
   CompressedRelationReader::ScanSpecAndBlocks mb{
       scanSpec, CompressedRelationReader::getRelevantBlocks(
-                    scanSpec, p.meta_.blockData())};
+                    scanSpec, p.getAugmentedMetadataForPermutation(
+                                  locatedTriplesSnapshot))};
 
   auto firstAndLastTriple = p.reader().getFirstAndLastTriple(
-      mb, getLocatedTriplesForPermutation(locatedTriplesSnapshot));
+      mb, p.getLocatedTriplesForPermutation(locatedTriplesSnapshot));
   if (!firstAndLastTriple.has_value()) {
     return std::nullopt;
   }
@@ -189,14 +194,14 @@ Permutation::IdTableGenerator Permutation::lazyScan(
   const auto& p = getActualPermutation(scanSpec);
   if (!blocks.has_value()) {
     auto blockSpan = CompressedRelationReader::getRelevantBlocks(
-        scanSpec, p.meta_.blockData());
+        scanSpec, p.getAugmentedMetadataForPermutation(locatedTriplesSnapshot));
     blocks = std::vector(blockSpan.begin(), blockSpan.end());
   }
   ColumnIndices columns{additionalColumns.begin(), additionalColumns.end()};
   return p.reader().lazyScan(
       scanSpec, std::move(blocks.value()), std::move(columns),
       std::move(cancellationHandle),
-      getLocatedTriplesForPermutation(locatedTriplesSnapshot), limitOffset);
+      p.getLocatedTriplesForPermutation(locatedTriplesSnapshot), limitOffset);
 }
 
 // ______________________________________________________________________
@@ -225,8 +230,28 @@ const Permutation& Permutation::getActualPermutation(Id id) const {
       ScanSpecification{id, std::nullopt, std::nullopt});
 }
 
+// TODO<joka921> The following two functions always assume that there were no
+// updates to the additional triples (which is technically true for now, because
+// we never modify the additional triples with the delta triples, because there
+// is some functionality missing for this. We have to fix this here and in the
+// `DeltaTriples` class.
+
 // ______________________________________________________________________
 const LocatedTriplesPerBlock& Permutation::getLocatedTriplesForPermutation(
     const LocatedTriplesSnapshot& locatedTriplesSnapshot) const {
-  return locatedTriplesSnapshot.getLocatedTriplesForPermutation(permutation_);
+  static const LocatedTriplesSnapshot emptySnapshot;
+  const auto& actualSnapshot =
+      isInternalPermutation_ ? emptySnapshot : locatedTriplesSnapshot;
+  return actualSnapshot.getLocatedTriplesForPermutation(permutation_);
+}
+
+// ______________________________________________________________________
+const std::vector<CompressedBlockMetadata>&
+Permutation::getAugmentedMetadataForPermutation(
+    const LocatedTriplesSnapshot& locatedTriplesSnapshot) const {
+  if (isInternalPermutation_) {
+    return meta_.blockData();
+  }
+  return getLocatedTriplesForPermutation(locatedTriplesSnapshot)
+      .getAugmentedMetadata();
 }
diff --git a/src/index/Permutation.h b/src/index/Permutation.h
@@ -161,8 +161,16 @@ class Permutation {
   const LocatedTriplesPerBlock& getLocatedTriplesForPermutation(
       const LocatedTriplesSnapshot& locatedTriplesSnapshot) const;
 
+  // From the given snapshot, get the augmented block metadata for this
+  // permutation.
+  const std::vector<CompressedBlockMetadata>&
+  getAugmentedMetadataForPermutation(
+      const LocatedTriplesSnapshot& locatedTriplesSnapshot) const;
+
   const CompressedRelationReader& reader() const { return reader_.value(); }
 
+  Enum permutation() const { return permutation_; }
+
  private:
   // Readable name for this permutation, e.g., `POS`.
   std::string readableName_;
@@ -185,4 +193,6 @@ class Permutation {
   std::unique_ptr<Permutation> internalPermutation_ = nullptr;
 
   std::function<bool(Id)> isInternalId_;
+
+  bool isInternalPermutation_ = false;
 };