Deduplicate the "other sets" in a LocalVocab

src/engine/LocalVocab.cpp

@@ -5,8 +5,6 @@
 
 #include "engine/LocalVocab.h"
 
-#include "absl/strings/str_cat.h"
-#include "global/Id.h"
 #include "global/ValueId.h"
 #include "util/TransparentFunctors.h"
 

src/engine/LocalVocab.h

@@ -5,6 +5,9 @@
 
 #pragma once
 
+#include <absl/container/flat_hash_set.h>
+#include <absl/container/node_hash_set.h>
+
 #include <algorithm>
 #include <cstdlib>
 #include <memory>
@@ -13,9 +16,7 @@
 #include <string>
 #include <vector>
 
-#include "absl/container/node_hash_set.h"
-#include "global/Id.h"
-#include "parser/LiteralOrIri.h"
+#include "index/LocalVocabEntry.h"
 #include "util/BlankNodeManager.h"
 #include "util/Exception.h"
 
@@ -43,7 +44,7 @@ class LocalVocab {
   std::shared_ptr<Set> primaryWordSet_ = std::make_shared<Set>();
 
   // The other sets of `LocalVocabEntry`s, which are static.
-  std::vector<std::shared_ptr<const Set>> otherWordSets_;
+  absl::flat_hash_set<std::shared_ptr<const Set>> otherWordSets_;
 
   // The number of words (so that we can compute `size()` in constant time).
   size_t size_ = 0;
@@ -114,12 +115,18 @@ class LocalVocab {
   // primary set of this `LocalVocab` remains unchanged.
   template <std::ranges::range R>
   void mergeWith(const R& vocabs) {
-    auto inserter = std::back_inserter(otherWordSets_);
     using std::views::filter;
+    auto addWordSet = [this](const std::shared_ptr<const Set>& set) {
+      bool added = otherWordSets_.insert(set).second;
+      size_ += static_cast<size_t>(added) * set->size();
+    };
+    // Note: Even though the `otherWordsSet_`is a hash set that filters out
+    // duplicates, we still manually filter out empty sets, because these
+    // typically don't compare equal to each other because of the`shared_ptr`
+    // semantics.
     for (const auto& vocab : vocabs | filter(std::not_fn(&LocalVocab::empty))) {
-      std::ranges::copy(vocab.otherWordSets_, inserter);
-      *inserter = vocab.primaryWordSet_;
-      size_ += vocab.size_;
+      std::ranges::for_each(vocab.otherWordSets_, addWordSet);
+      addWordSet(vocab.primaryWordSet_);
     }
 
     // Also merge the `vocabs` `LocalBlankNodeManager`s, if they exist.

src/parser/LiteralOrIri.h

@@ -4,13 +4,10 @@
 
 #pragma once
 
-#include <absl/strings/str_cat.h>
-
 #include <variant>
 
 #include "parser/Iri.h"
 #include "parser/Literal.h"
-#include "util/Exception.h"
 
 namespace ad_utility::triple_component {
 static constexpr char literalPrefixChar = '"';

test/LocalVocabTest.cpp

@@ -410,3 +410,45 @@ TEST(LocalVocab, getBlankNodeIndex) {
   BlankNodeIndex b = v.getBlankNodeIndex(&bnm);
   EXPECT_NE(a, b);
 }
+
+// _____________________________________________________________________________
+TEST(LocalVocab, otherWordSetIsTransitivelyPropagated) {
+  using ad_utility::triple_component::LiteralOrIri;
+  LocalVocab original;
+  original.getIndexAndAddIfNotContained(
+      LocalVocabEntry{LiteralOrIri::literalWithoutQuotes("test")});
+
+  LocalVocab clone = original.clone();
+  LocalVocab mergeCandidate;
+  mergeCandidate.mergeWith(std::span{&clone, 1});
+
+  EXPECT_EQ(mergeCandidate.size(), 1);
+  EXPECT_THAT(mergeCandidate.getAllWordsForTesting(),
+              ::testing::UnorderedElementsAre(
+                  LiteralOrIri::literalWithoutQuotes("test")));
+}
+
+// _____________________________________________________________________________
+TEST(LocalVocab, sizeIsProperlyUpdatedOnMerge) {
+  using ad_utility::triple_component::LiteralOrIri;
+  using ::testing::UnorderedElementsAre;
+  LocalVocab original;
+  original.getIndexAndAddIfNotContained(
+      LocalVocabEntry{LiteralOrIri::literalWithoutQuotes("test")});
+
+  LocalVocab clone1 = original.clone();
+  LocalVocab clone2 = original.clone();
+  clone2.mergeWith(std::span{&original, 1});
+  original.mergeWith(std::span{&clone1, 1});
+
+  // Implementation detail, merging does add to the "other word set" but does
+  // not deduplicate with the primary word set.
+  EXPECT_EQ(original.size(), 2);
+  EXPECT_THAT(original.getAllWordsForTesting(),
+              UnorderedElementsAre(LiteralOrIri::literalWithoutQuotes("test"),
+                                   LiteralOrIri::literalWithoutQuotes("test")));
+
+  EXPECT_EQ(clone2.size(), 1);
+  EXPECT_THAT(clone2.getAllWordsForTesting(),
+              UnorderedElementsAre(LiteralOrIri::literalWithoutQuotes("test")));
+}