Minor updates regarding less memory usage, also fixing a resource leak

sirixdb · Sep 19, 2024 · e34784d · e34784d
1 parent 1aaafd1
commit e34784d
Show file tree

Hide file tree

Showing 9 changed files with 72 additions and 29 deletions.
diff --git a/build.gradle b/build.gradle
@@ -264,7 +264,7 @@ subprojects {
                  "-XX:+DebugNonSafepoints",
                  //"-XX:+UseShenandoahGC",
                  "-Xlog:gc*=debug:file=g1.log",
-//                 "-XX:+UseZGC",
+                 "-XX:+UseZGC",
 //                 "-XX:+ZGenerational",
                  "-verbose:gc",
                  "-XX:+HeapDumpOnOutOfMemoryError",
@@ -285,7 +285,7 @@ subprojects {
                 /* "-XX:MaxInlineSize=500" */])
 
         minHeapSize = '5g'
-        maxHeapSize = '8g'
+        maxHeapSize = '12g'
     }
 }
 

diff --git a/bundles/sirix-core/src/main/java/io/sirix/access/LocalDatabase.java b/bundles/sirix-core/src/main/java/io/sirix/access/LocalDatabase.java
@@ -119,7 +119,7 @@ public LocalDatabase(final TransactionManager transactionManager, final Database
   }
 
   private void addResourceToBufferManagerMapping(Path resourceFile, ResourceConfiguration resourceConfig) {
-    bufferManagers.put(resourceFile, new BufferManagerImpl(1_000_000, 65_536 * 1_0000, 5_000, 50_000, 500, 20));
+    bufferManagers.put(resourceFile, new BufferManagerImpl(500_000, 65_536 * 100, 5_000, 50_000, 500, 20));
   }
 
   @Override

diff --git a/bundles/sirix-core/src/main/java/io/sirix/access/trx/node/HashType.java b/bundles/sirix-core/src/main/java/io/sirix/access/trx/node/HashType.java
@@ -19,6 +19,6 @@ public static HashType fromString(String string) {
         return hashType;
       }
     }
-    throw new IllegalArgumentException(STR."No constant with name \{string} found");
+    throw new IllegalArgumentException("No constant with name " + string + " found");
   }
 }
diff --git a/bundles/sirix-core/src/main/java/io/sirix/access/trx/page/NodePageReadOnlyTrx.java b/bundles/sirix-core/src/main/java/io/sirix/access/trx/page/NodePageReadOnlyTrx.java
@@ -70,8 +70,6 @@ private record RecordPage(int index, IndexType indexType, long recordPageKey, in
                             PageReference pageReference, KeyValueLeafPage page) {
   }
 
-  private final PageReference pageReferenceForUnpinning = new PageReference();
-
   /**
    * Page reader exclusively assigned to this transaction.
    */
@@ -527,9 +525,14 @@ private void setMostRecentlyReadRecordPage(@NonNull IndexLogKey indexLogKey, @No
         if (trxIntentLog == null) {
           if (resourceBufferManager.getRecordPageCache().get(pathSummaryRecordPage.pageReference) != null) {
             assert !pathSummaryRecordPage.page.isClosed();
-            pathSummaryRecordPage.page.decrementPinCount();
-            resourceBufferManager.getRecordPageCache()
-                                 .put(pathSummaryRecordPage.pageReference, pathSummaryRecordPage.page);
+            resourceBufferManager.getRecordPageCache().get(pathSummaryRecordPage.pageReference, (_, _) -> {
+              var kvPage = pathSummaryRecordPage.page;
+              kvPage.decrementPinCount();
+              return kvPage;
+            });
+//            pathSummaryRecordPage.page.decrementPinCount();
+//            resourceBufferManager.getRecordPageCache()
+//                                 .put(pathSummaryRecordPage.pageReference, pathSummaryRecordPage.page);
           }
         } else {
           pathSummaryRecordPage.pageReference.setPage(null);
@@ -549,10 +552,15 @@ private void setMostRecentlyReadRecordPage(@NonNull IndexLogKey indexLogKey, @No
       if (secondMostRecentlyReadRecordPage != null
           && resourceBufferManager.getRecordPageCache().get(secondMostRecentlyReadRecordPage.pageReference) != null) {
         assert !secondMostRecentlyReadRecordPage.page.isClosed();
-        secondMostRecentlyReadRecordPage.page.decrementPinCount();
-        resourceBufferManager.getRecordPageCache()
-                             .put(secondMostRecentlyReadRecordPage.pageReference,
-                                  secondMostRecentlyReadRecordPage.page);
+        resourceBufferManager.getRecordPageCache().get(secondMostRecentlyReadRecordPage.pageReference, (_, _) -> {
+          var kvPage = secondMostRecentlyReadRecordPage.page;
+          kvPage.decrementPinCount();
+          return kvPage;
+        });
+//        secondMostRecentlyReadRecordPage.page.decrementPinCount();
+//        resourceBufferManager.getRecordPageCache()
+//                             .put(secondMostRecentlyReadRecordPage.pageReference,
+//                                  secondMostRecentlyReadRecordPage.page);
       }
       assert mostRecentlyReadRecordPage == null || !mostRecentlyReadRecordPage.page.isClosed();
       secondMostRecentlyReadRecordPage = mostRecentlyReadRecordPage;
@@ -603,14 +611,17 @@ private void unpinPageFragments(PageReference pageReference, List<KeyValuePage<D
     }
 
     var mostRecentPageFragment = pages.getFirst();
-    resourceBufferManager.getPageCache().put(pageReference, mostRecentPageFragment);
+    resourceBufferManager.getPageCache().get(pageReference, (_, _) -> {
+      mostRecentPageFragment.decrementPinCount();
+      return mostRecentPageFragment;
+    });
 
     var pageFragments = pageReference.getPageFragments();
     for (int i = 1; i < pages.size(); i++) {
       var pageFragment = pages.get(i);
       pageFragment.decrementPinCount();
       var pageFragmentKey = pageFragments.get(i - 1);
-      resourceBufferManager.getPageCache().put(new PageReference().setKey(pageFragmentKey.key()), pageFragment);
+      resourceBufferManager.getPageCache().get(new PageReference().setKey(pageFragmentKey.key()), (_, _) -> pageFragment);
     }
   }
 
@@ -621,9 +632,11 @@ private Page getInMemoryPageInstance(@NonNull IndexLogKey indexLogKey,
 
     if (page != null) {
       if (trxIntentLog == null || indexLogKey.getIndexType() != IndexType.PATH_SUMMARY) {
-        var kvLeafPage = ((KeyValueLeafPage) page);
-        kvLeafPage.incrementPinCount();
-        resourceBufferManager.getRecordPageCache().put(pageReferenceToRecordPage, kvLeafPage);
+        resourceBufferManager.getPageCache().get(pageReferenceToRecordPage, (_, _) -> {
+          var kvLeafPage = ((KeyValueLeafPage) page);
+          kvLeafPage.incrementPinCount();
+          return kvLeafPage;
+        });
       }
       setMostRecentlyReadRecordPage(indexLogKey, pageReferenceToRecordPage, (KeyValueLeafPage) page);
       return page;

diff --git a/bundles/sirix-core/src/main/java/io/sirix/cache/TransactionIntentLog.java b/bundles/sirix-core/src/main/java/io/sirix/cache/TransactionIntentLog.java
@@ -78,6 +78,10 @@ public void put(final PageReference key, final PageContainer value) {
    */
   public void clear() {
     logKey = 0;
+    for (final PageContainer pageContainer : list) {
+      pageContainer.getComplete().clear();
+      pageContainer.getModified().clear();
+    }
     list.clear();
   }
 

diff --git a/bundles/sirix-core/src/main/java/io/sirix/page/KeyValueLeafPage.java b/bundles/sirix-core/src/main/java/io/sirix/page/KeyValueLeafPage.java
@@ -378,7 +378,7 @@ private static int alignOffset(int offset) {
 
   @Override
   public void setSlot(byte[] recordData, int slotNumber) {
-    setData(MemorySegment.ofArray(recordData), slotNumber, slotOffsets, slotMemory);
+    setData(recordData, slotNumber, slotOffsets, slotMemory);
   }
 
   // For testing.
@@ -391,12 +391,29 @@ public void setSlot(MemorySegment data, int slotNumber) {
     setData(data, slotNumber, slotOffsets, slotMemory);
   }
 
-  private MemorySegment setData(MemorySegment data, int slotNumber, int[] offsets, MemorySegment memory) {
-    if (data == null || data.byteSize() == 0) {
+  private MemorySegment setData(Object data, int slotNumber, int[] offsets, MemorySegment memory) {
+    if (data == null) {
       return null;
     }
 
-    var dataSize = (int) data.byteSize();
+    int dataSize;
+
+    if (data instanceof MemorySegment) {
+      dataSize = (int) ((MemorySegment) data).byteSize();
+
+      if (dataSize == 0) {
+        return null;
+      }
+    } else if (data instanceof byte[]) {
+      dataSize = ((byte[]) data).length;
+
+      if (dataSize == 0) {
+        return null;
+      }
+    } else {
+      throw new IllegalArgumentException("Data must be either a MemorySegment or a byte array.");
+    }
+
     int requiredSize = INT_SIZE + dataSize;
     int currentOffset = offsets[slotNumber];
 
@@ -423,7 +440,11 @@ private MemorySegment setData(MemorySegment data, int slotNumber, int[] offsets,
       if (currentSize == requiredSize) {
         // If the size is the same, update it directly.
         memory.set(ValueLayout.JAVA_INT, alignedOffset, dataSize);
-        memory.asSlice(alignedOffset + INT_SIZE, dataSize).copyFrom(data);
+        if (data instanceof MemorySegment) {
+          MemorySegment.copy((MemorySegment) data, 0, memory, alignedOffset + INT_SIZE, dataSize);
+        } else {
+          MemorySegment.copy(data, 0, memory, ValueLayout.JAVA_BYTE, alignedOffset + INT_SIZE, dataSize);
+        }
 
         return null; // No resizing needed
       } else {
@@ -433,7 +454,6 @@ private MemorySegment setData(MemorySegment data, int slotNumber, int[] offsets,
     } else {
       // If the slot is empty, determine where to place the new data.
       currentOffset = findFreeSpaceForSlots(requiredSize, isSlotMemory);
-      //currentOffset = findFreeSpace(offsets, memory, requiredSize, (int) memory.byteSize());
       offsets[slotNumber] = alignOffset(currentOffset);
       updateLastSlotIndex(slotNumber, isSlotMemory);
     }
@@ -446,7 +466,12 @@ private MemorySegment setData(MemorySegment data, int slotNumber, int[] offsets,
     // Write the new data into the slot.
     int alignedOffset = alignOffset(currentOffset);
     memory.set(ValueLayout.JAVA_INT, alignedOffset, dataSize);
-    memory.asSlice(alignedOffset + INT_SIZE, dataSize).copyFrom(data);
+
+    if (data instanceof MemorySegment) {
+      MemorySegment.copy((MemorySegment) data, 0, memory, alignedOffset + INT_SIZE, dataSize);
+    } else {
+      MemorySegment.copy(data, 0, memory, ValueLayout.JAVA_BYTE, alignedOffset + INT_SIZE, dataSize);
+    }
 
     // Update slotMemoryFreeSpaceStart after adding the slot.
     updateFreeSpaceStart(offsets, memory, isSlotMemory);
@@ -734,7 +759,7 @@ private int getNumberOfNonNullEntries(final DataRecord[] entries) {
     int count = 0;
     for (int i = 0; i < Constants.NDP_NODE_COUNT; i++) {
       final DataRecord record = entries[i];
-      if (record != null || getSlot(i) != null) {
+      if (record != null || isSlotSet(i)) {
         count++;
       }
     }

diff --git a/bundles/sirix-core/src/main/java/io/sirix/page/PageKind.java b/bundles/sirix-core/src/main/java/io/sirix/page/PageKind.java
@@ -180,6 +180,7 @@ public void serializePage(final ResourceConfiguration resourceConfig, final Byte
 
       if (bytes != null) {
         sink.write(bytes.bytesForWrite());
+        bytes.clear();
         return;
       }
 

diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.8-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.10-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

diff --git a/libraries.gradle b/libraries.gradle
@@ -54,8 +54,8 @@ testLibraries = [
         junitVintageEngine       : 'org.junit.vintage:junit-vintage-engine:5.8.1',
         junitPlatformLauncher    : 'org.junit.platform:junit-platform-launcher:1.8.1',
         junitPlatformRunner      : 'org.junit.platform:junit-platform-runner:1.8.1',
-        mockitoCore              : 'org.mockito:mockito-core:5.11.0',
-        byteBuddy                : 'net.bytebuddy:byte-buddy:1.14.8',
+        mockitoCore              : 'org.mockito:mockito-core:5.13.0',
+        byteBuddy                : 'net.bytebuddy:byte-buddy:1.15.1',
         brackit                  : 'io.sirix:brackit:0.4:tests',
         kotlinTestJunit          : 'org.jetbrains.kotlin:kotlin-test-junit:1.5.31',
         junit                    : 'junit:junit:4.13.2',
-Original file line number
+Diff line change
@@ Expand Up @@
           if (bytes != null) {
             sink.write(bytes.bytesForWrite());
+            bytes.clear();
             return;
           }
@@ Expand Down @@