Extend DeltaTriples to also keep track of updates to the internal permutations

src/index/DeltaTriples.cpp

@@ -22,7 +22,9 @@
 #include "util/Serializer/TripleSerializer.h"
 
 // ____________________________________________________________________________
-LocatedTriples::iterator& DeltaTriples::LocatedTripleHandles::forPermutation(
+template <bool internal>
+LocatedTriples::iterator&
+DeltaTriples::LocatedTripleHandles<internal>::forPermutation(
     Permutation::Enum permutation) {
   return handles_[static_cast<size_t>(permutation)];
 }
@@ -31,37 +33,54 @@
 void DeltaTriples::clear() {
   triplesInserted_.clear();
   triplesDeleted_.clear();
+  internalTriplesInserted_.clear();
+  internalTriplesDeleted_.clear();
   ql::ranges::for_each(locatedTriples(), &LocatedTriplesPerBlock::clear);
+  ql::ranges::for_each(internalLocatedTriples_, &LocatedTriplesPerBlock::clear);
 }
 
 // ____________________________________________________________________________
-std::vector<DeltaTriples::LocatedTripleHandles>
+template <bool internal>
+auto& DeltaTriples::getLocatedTriple() {
+  if constexpr (internal) {
+    return internalLocatedTriples_;
+  } else {
+    return locatedTriples();
+  }
+}
+
+// ____________________________________________________________________________
+template <bool internal>
+std::vector<DeltaTriples::LocatedTripleHandles<internal>>
 DeltaTriples::locateAndAddTriples(CancellationHandle cancellationHandle,
                                   ql::span<const IdTriple<0>> triples,
                                   bool insertOrDelete,
                                   ad_utility::timer::TimeTracer& tracer) {
-  std::array<std::vector<LocatedTriples::iterator>, Permutation::ALL.size()>
+  constexpr const auto& allPermutations =
+      internal ? Permutation::INTERNAL : Permutation::ALL;
+  auto& lt = getLocatedTriple<internal>();
+  std::array<std::vector<LocatedTriples::iterator>, allPermutations.size()>
       intermediateHandles;
-  for (auto permutation : Permutation::ALL) {
+  for (auto permutation : allPermutations) {
     tracer.beginTrace(std::string{Permutation::toString(permutation)});
     tracer.beginTrace("locateTriples");
-    auto& perm = index_.getPermutation(permutation);
+    auto& basePerm = index_.getPermutation(permutation);
+    auto& perm = internal ? basePerm.internalPermutation() : basePerm;
     auto locatedTriples = LocatedTriple::locateTriplesInPermutation(
         triples, perm.metaData().blockData(), perm.keyOrder(), insertOrDelete,
         cancellationHandle);
     cancellationHandle->throwIfCancelled();
     tracer.endTrace("locateTriples");
     tracer.beginTrace("addToLocatedTriples");
     intermediateHandles[static_cast<size_t>(permutation)] =
-        this->locatedTriples()[static_cast<size_t>(permutation)].add(
-            locatedTriples, tracer);
+        lt[static_cast<size_t>(permutation)].add(locatedTriples, tracer);
     cancellationHandle->throwIfCancelled();
     tracer.endTrace("addToLocatedTriples");
     tracer.endTrace(Permutation::toString(permutation));
   }
   tracer.beginTrace("transformHandles");
-  std::vector<DeltaTriples::LocatedTripleHandles> handles{triples.size()};
-  for (auto permutation : Permutation::ALL) {
+  std::vector<LocatedTripleHandles<internal>> handles{triples.size()};
+  for (auto permutation : allPermutations) {
     for (size_t i = 0; i < triples.size(); i++) {
       handles[i].forPermutation(permutation) =
           intermediateHandles[static_cast<size_t>(permutation)][i];
@@ -72,12 +91,16 @@
 }
 
 // ____________________________________________________________________________
-void DeltaTriples::eraseTripleInAllPermutations(LocatedTripleHandles& handles) {
+template <bool internal>
+void DeltaTriples::eraseTripleInAllPermutations(
+    LocatedTripleHandles<internal>& handles) {
+  constexpr const auto& allPermutations =
+      internal ? Permutation::INTERNAL : Permutation::ALL;
+  auto& lt = getLocatedTriple<internal>();
   // Erase for all permutations.
-  for (auto permutation : Permutation::ALL) {
+  for (auto permutation : allPermutations) {
     auto ltIter = handles.forPermutation(permutation);
-    locatedTriples()[static_cast<int>(permutation)].erase(ltIter->blockIndex_,
-                                                          ltIter);
+    lt[static_cast<int>(permutation)].erase(ltIter->blockIndex_, ltIter);
   }
 }
 
@@ -108,6 +131,30 @@
                     triplesDeleted_, triplesInserted_, tracer);
 }
 
+// ____________________________________________________________________________
+void DeltaTriples::insertInternalTriples(
+    CancellationHandle cancellationHandle, Triples triples,
+    ad_utility::timer::TimeTracer& tracer) {
+  AD_LOG_DEBUG << "Inserting"
+               << " " << triples.size()
+               << " internal triples (including idempotent triples)."
+               << std::endl;
+  modifyTriplesImpl(std::move(cancellationHandle), std::move(triples), true,
+                    internalTriplesInserted_, internalTriplesDeleted_, tracer);
+}
+
+// ____________________________________________________________________________
+void DeltaTriples::deleteInternalTriples(
+    CancellationHandle cancellationHandle, Triples triples,
+    ad_utility::timer::TimeTracer& tracer) {
+  AD_LOG_DEBUG << "Deleting"
+               << " " << triples.size()
+               << " internal triples (including idempotent triples)."
+               << std::endl;
+  modifyTriplesImpl(std::move(cancellationHandle), std::move(triples), false,
+                    internalTriplesDeleted_, internalTriplesInserted_, tracer);
+}
+
 // ____________________________________________________________________________
 void DeltaTriples::rewriteLocalVocabEntriesAndBlankNodes(Triples& triples) {
   // Remember which original blank node (from the parsing of an insert
@@ -164,10 +211,11 @@
 }
 
 // ____________________________________________________________________________
+template <bool internal>
 void DeltaTriples::modifyTriplesImpl(CancellationHandle cancellationHandle,
                                      Triples triples, bool insertOrDelete,
-                                     TriplesToHandlesMap& targetMap,
-                                     TriplesToHandlesMap& inverseMap,
+                                     TriplesToHandlesMap<internal>& targetMap,
+                                     TriplesToHandlesMap<internal>& inverseMap,
                                      ad_utility::timer::TimeTracer& tracer) {
   tracer.beginTrace("rewriteLocalVocabEntries");
   rewriteLocalVocabEntriesAndBlankNodes(triples);
@@ -195,8 +243,9 @@
   tracer.endTrace("removeInverseTriples");
   tracer.beginTrace("locatedAndAdd");
 
-  std::vector<LocatedTripleHandles> handles = locateAndAddTriples(
-      std::move(cancellationHandle), triples, insertOrDelete, tracer);
+  std::vector<LocatedTripleHandles<internal>> handles =
+      locateAndAddTriples<internal>(std::move(cancellationHandle), triples,
+                                    insertOrDelete, tracer);
   tracer.endTrace("locatedAndAdd");
   tracer.beginTrace("markTriples");
 
@@ -215,6 +264,15 @@
   return locatedTriplesPerBlock_[static_cast<int>(permutation)];
 }
 
+// ____________________________________________________________________________
+const LocatedTriplesPerBlock&
+LocatedTriplesSnapshot::getInternalLocatedTriplesForPermutation(
+    Permutation::Enum permutation) const {
+  AD_CONTRACT_CHECK(permutation == Permutation::PSO ||
+                    permutation == Permutation::POS);
+  return internalLocatedTriplesPerBlock_[static_cast<int>(permutation)];
+}
+
 // ____________________________________________________________________________
 SharedLocatedTriplesSnapshot DeltaTriples::getSnapshot() {
   // NOTE: Both members of the `LocatedTriplesSnapshot` are copied, but the
@@ -224,7 +282,8 @@
   ++nextSnapshotIndex_;
   return SharedLocatedTriplesSnapshot{
       std::make_shared<LocatedTriplesSnapshot>(LocatedTriplesSnapshot{
-          locatedTriples(), localVocab_.getLifetimeExtender(), snapshotIndex})};
+          locatedTriples(), internalLocatedTriples_,
+          localVocab_.getLifetimeExtender(), snapshotIndex})};
 }
 
 // ____________________________________________________________________________
@@ -328,24 +387,33 @@
 // _____________________________________________________________________________
 void DeltaTriples::setOriginalMetadata(
     Permutation::Enum permutation,
-    std::shared_ptr<const std::vector<CompressedBlockMetadata>> metadata) {
-  locatedTriples()
-      .at(static_cast<size_t>(permutation))
-      .setOriginalMetadata(std::move(metadata));
+    std::shared_ptr<const std::vector<CompressedBlockMetadata>> metadata,
+    bool setInternalMetadata) {
+  auto& locatedTriplesPerBlock =
+      setInternalMetadata
+          ? internalLocatedTriples_.at(static_cast<size_t>(permutation))
+          : locatedTriples().at(static_cast<size_t>(permutation));
+  locatedTriplesPerBlock.setOriginalMetadata(std::move(metadata));
 }
 
 // _____________________________________________________________________________
 void DeltaTriples::updateAugmentedMetadata() {
   ql::ranges::for_each(locatedTriples(),
                        &LocatedTriplesPerBlock::updateAugmentedMetadata);
+  ql::ranges::for_each(internalLocatedTriples_,
+                       &LocatedTriplesPerBlock::updateAugmentedMetadata);
 }
 
 // _____________________________________________________________________________
 void DeltaTriples::writeToDisk() const {
   if (!filenameForPersisting_.has_value()) {
     return;
   }
-  auto toRange = [](const TriplesToHandlesMap& map) {
+  // TODO<RobinTF> Currently this only writes non-internal delta triples to
+  // disk. The internal triples will be regenerated when importing the rest
+  // again. In the future we might to also want to explicitly store the internal
+  // triples.
+  auto toRange = [](const TriplesToHandlesMap<false>& map) {
     return map | ql::views::keys |
            ql::views::transform(
                [](const IdTriple<0>& triple) -> const std::array<Id, 4>& {

src/index/DeltaTriples.h

@@ -26,13 +26,16 @@
 // permutations.
 using LocatedTriplesPerBlockAllPermutations =
     std::array<LocatedTriplesPerBlock, Permutation::ALL.size()>;
+using LocatedTriplesPerBlockInternalPermutations =
+    std::array<LocatedTriplesPerBlock, Permutation::INTERNAL.size()>;
 
 // The locations of a set of delta triples (triples that were inserted or
 // deleted since the index was built) in each of the six permutations, and a
 // local vocab. This is all the information that is required to perform a query
 // that correctly respects these delta triples, hence the name.
 struct LocatedTriplesSnapshot {
   LocatedTriplesPerBlockAllPermutations locatedTriplesPerBlock_;
+  LocatedTriplesPerBlockInternalPermutations internalLocatedTriplesPerBlock_;
   // Make sure to keep the local vocab alive as long as the snapshot is alive.
   // The `DeltaTriples` class may concurrently add new entries under the hood,
   // but this is safe because the `LifetimeExtender` prevents access entirely.
@@ -42,6 +45,9 @@ struct LocatedTriplesSnapshot {
   // Get `TripleWithPosition` objects for given permutation.
   const LocatedTriplesPerBlock& getLocatedTriplesForPermutation(
       Permutation::Enum permutation) const;
+  // Get `TripleWithPosition` objects for given internal permutation.
+  const LocatedTriplesPerBlock& getInternalLocatedTriplesForPermutation(
+      Permutation::Enum permutation) const;
 };
 
 // A shared pointer to a constant `LocatedTriplesSnapshot`, but as an explicit
@@ -98,6 +104,9 @@ class DeltaTriples {
   // The located triples for all the 6 permutations.
   LocatedTriplesPerBlockAllPermutations locatedTriples_;
 
+  // The located triples for the 2 internal permutations.
+  LocatedTriplesPerBlockInternalPermutations internalLocatedTriples_;
+
   // The local vocabulary of the delta triples (they may have components,
   // which are not contained in the vocabulary of the original index).
   LocalVocab localVocab_;
@@ -118,21 +127,26 @@ class DeltaTriples {
 
   // Each delta triple needs to know where it is stored in each of the six
   // `LocatedTriplesPerBlock` above.
+  template <bool internal>
   struct LocatedTripleHandles {
     using It = LocatedTriples::iterator;
-    std::array<It, Permutation::ALL.size()> handles_;
+    std::array<It, (internal ? Permutation::INTERNAL : Permutation::ALL).size()>
+        handles_;
 
     LocatedTriples::iterator& forPermutation(Permutation::Enum permutation);
   };
+  template <bool internal>
   using TriplesToHandlesMap =
-      ad_utility::HashMap<IdTriple<0>, LocatedTripleHandles>;
+      ad_utility::HashMap<IdTriple<0>, LocatedTripleHandles<internal>>;
 
   // The sets of triples added to and subtracted from the original index. Any
   // triple can be at most in one of the sets. The information whether a triple
   // is in the index is missing. This means that a triple that is in the index
   // may still be in the inserted set and vice versa.
-  TriplesToHandlesMap triplesInserted_;
-  TriplesToHandlesMap triplesDeleted_;
+  TriplesToHandlesMap<false> triplesInserted_;
+  TriplesToHandlesMap<false> triplesDeleted_;
+  TriplesToHandlesMap<true> internalTriplesInserted_;
+  TriplesToHandlesMap<true> internalTriplesDeleted_;
 
  public:
   // Construct for given index.
@@ -170,6 +184,14 @@ class DeltaTriples {
   }
   DeltaTriplesCount getCounts() const;
 
+  // The number of internal delta triples added and subtracted.
+  int64_t numInternalInserted() const {
+    return static_cast<int64_t>(internalTriplesInserted_.size());
+  }
+  int64_t numInternalDeleted() const {
+    return static_cast<int64_t>(internalTriplesDeleted_.size());
+  }
+
   // Insert triples.
   void insertTriples(CancellationHandle cancellationHandle, Triples triples,
                      ad_utility::timer::TimeTracer& tracer =
@@ -180,6 +202,20 @@ class DeltaTriples {
                      ad_utility::timer::TimeTracer& tracer =
                          ad_utility::timer::DEFAULT_TIME_TRACER);
 
+  // Insert internal delta triples for efficient language filters and patterns.
+  // Currently only used by test code.
+  void insertInternalTriples(CancellationHandle cancellationHandle,
+                             Triples triples,
+                             ad_utility::timer::TimeTracer& tracer =
+                                 ad_utility::timer::DEFAULT_TIME_TRACER);
+
+  // Delete triplesdelta triples for efficient language filters and patterns.
+  // Currently only used by test code.
+  void deleteInternalTriples(CancellationHandle cancellationHandle,
+                             Triples triples,
+                             ad_utility::timer::TimeTracer& tracer =
+                                 ad_utility::timer::DEFAULT_TIME_TRACER);
+
   // If the `filename` is set, then `writeToDisk()` will write these
   // `DeltaTriples` to `filename.value()`. If `filename` is `nullopt`, then
   // `writeToDisk` will be a nullop.
@@ -197,21 +233,29 @@ class DeltaTriples {
   SharedLocatedTriplesSnapshot getSnapshot();
 
   // Register the original `metadata` for the given `permutation`. This has to
-  // be called before any updates are processed.
+  // be called before any updates are processed. If `setInternalMetadata` is
+  // true, this will set the metadata to the internal permutations instead.
   void setOriginalMetadata(
       Permutation::Enum permutation,
-      std::shared_ptr<const std::vector<CompressedBlockMetadata>> metadata);
+      std::shared_ptr<const std::vector<CompressedBlockMetadata>> metadata,
+      bool setInternalMetadata);
 
   // Update the block metadata.
   void updateAugmentedMetadata();
 
  private:
+  // Helper function to get the correct located triple (either internal or
+  // external), depending on the `internal` template parameter.
+  template <bool internal>
+  auto& getLocatedTriple();
+
   // Find the position of the given triple in the given permutation and add it
   // to each of the six `LocatedTriplesPerBlock` maps (one per permutation).
   // When `insertOrDelete` is `true`, the triples are inserted, otherwise
   // deleted. Return the iterators of where it was added (so that we can easily
   // delete it again from these maps later).
-  std::vector<LocatedTripleHandles> locateAndAddTriples(
+  template <bool internal>
+  std::vector<LocatedTripleHandles<internal>> locateAndAddTriples(
       CancellationHandle cancellationHandle,
       ql::span<const IdTriple<0>> triples, bool insertOrDelete,
       ad_utility::timer::TimeTracer& tracer =
@@ -223,9 +267,11 @@ class DeltaTriples {
   // triples. When `insertOrDelete` is `false`, the triples are deleted, and it
   // is the other way around:. This is used to resolve insertions or deletions
   // that are idempotent or cancel each other out.
+  template <bool internal>
   void modifyTriplesImpl(CancellationHandle cancellationHandle, Triples triples,
-                         bool shouldExist, TriplesToHandlesMap& targetMap,
-                         TriplesToHandlesMap& inverseMap,
+                         bool shouldExist,
+                         TriplesToHandlesMap<internal>& targetMap,
+                         TriplesToHandlesMap<internal>& inverseMap,
                          ad_utility::timer::TimeTracer& tracer =
                              ad_utility::timer::DEFAULT_TIME_TRACER);
 
@@ -246,7 +292,8 @@ class DeltaTriples {
   // NOTE: The iterators are invalid afterward. That is OK, as long as we also
   // delete the respective entry in `triplesInserted_` or `triplesDeleted_`,
   // which stores these iterators.
-  void eraseTripleInAllPermutations(LocatedTripleHandles& handles);
+  template <bool internal>
+  void eraseTripleInAllPermutations(LocatedTripleHandles<internal>& handles);
 
   friend class DeltaTriplesManager;
 };

src/index/IndexImpl.cpp

@@ -946,17 +946,13 @@ void IndexImpl::createFromOnDiskIndex(const std::string& onDiskBase,
 
   // Load the permutations and register the original metadata for the delta
   // triples.
-  // TODO<joka921> We could delegate the setting of the metadata to the
-  // `Permutation`class, but we first have to deal with The delta triples for
-  // the additional permutations.
   // The setting of the metadata doesn't affect the contents of the delta
   // triples, so we don't need to call `writeToDisk`, therefore the second
   // argument to `modify` is `false`.
-  auto setMetadata = [this](const Permutation& p) {
+  auto setMetadata = [this](const Permutation& permutation) {
     deltaTriplesManager().modify<void>(
-        [&p](DeltaTriples& deltaTriples) {
-          deltaTriples.setOriginalMetadata(p.permutation(),
-                                           p.metaData().blockDataShared());
+        [&permutation](DeltaTriples& deltaTriples) {
+          permutation.setOriginalMetadataForDeltaTriples(deltaTriples);
         },
         false, false);
   };