tvlist_sort_threshold config property

apache · Dec 7, 2024 · 73e9f83 · 73e9f83
1 parent e3d2d5e
commit 73e9f83
Show file tree

Hide file tree

Showing 5 changed files with 27 additions and 4 deletions.
diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java
@@ -430,6 +430,9 @@ public class IoTDBConfig {
   /** The sort algorithm used in TVList */
   private TVListSortAlgorithm tvListSortAlgorithm = TVListSortAlgorithm.TIM;
 
+  /** the threshold when working TVList is sorted and handover in writable memtable */
+  private int tvListSortThreshold = 0;
+
   /** When average series point number reaches this, flush the memtable to disk */
   private int avgSeriesPointNumberThreshold = 100000;
 
@@ -2309,6 +2312,14 @@ public void setTvListSortAlgorithm(TVListSortAlgorithm tvListSortAlgorithm) {
     this.tvListSortAlgorithm = tvListSortAlgorithm;
   }
 
+  public int getTvListSortThreshold() {
+    return tvListSortThreshold;
+  }
+
+  public void setTVListSortThreshold(int tvListSortThreshold) {
+    this.tvListSortThreshold = tvListSortThreshold;
+  }
+
   public int getAvgSeriesPointNumberThreshold() {
     return avgSeriesPointNumberThreshold;
   }

diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java
@@ -442,6 +442,14 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException
                 .map(String::trim)
                 .orElse(conf.getTvListSortAlgorithm().toString())));
 
+    conf.setTVListSortThreshold(
+        Integer.parseInt(
+            Optional.ofNullable(
+                    properties.getProperty(
+                        "tvlist_sort_threshold", Integer.toString(conf.getTvListSortThreshold())))
+                .map(String::trim)
+                .orElse(Integer.toString(conf.getTvListSortThreshold()))));
+
     conf.setAvgSeriesPointNumberThreshold(
         Integer.parseInt(
             Optional.ofNullable(

diff --git a/...rc/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java b/...rc/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java
@@ -19,6 +19,7 @@
 package org.apache.iotdb.db.storageengine.dataregion.memtable;
 
 import org.apache.iotdb.common.rpc.thrift.TSStatus;
+import org.apache.iotdb.db.conf.IoTDBDescriptor;
 import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue;
 import org.apache.iotdb.db.utils.datastructure.TVList;
 
@@ -31,8 +32,7 @@
 import java.util.List;
 
 public interface IWritableMemChunk extends WALEntryValue {
-  // TODO: read from configuration file
-  int SORT_THRESHOLD = 5000;
+  int TVLIST_SORT_THRESHOLD = IoTDBDescriptor.getInstance().getConfig().getTvListSortThreshold();
 
   boolean putLongWithFlushCheck(long t, long v);
 

diff --git a/...src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java b/...src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java
@@ -129,7 +129,7 @@ public boolean writeWithFlushCheck(long insertTime, Object objectValue) {
       return true;
     }
 
-    if (list.rowCount() >= SORT_THRESHOLD) {
+    if (TVLIST_SORT_THRESHOLD > 0 && list.rowCount() >= TVLIST_SORT_THRESHOLD) {
       handoverTvList();
     }
     return false;
@@ -180,7 +180,7 @@ public boolean writeWithFlushCheck(
       return true;
     }
 
-    if (list.rowCount() >= SORT_THRESHOLD) {
+    if (TVLIST_SORT_THRESHOLD > 0 && list.rowCount() >= TVLIST_SORT_THRESHOLD) {
       handoverTvList();
     }
     return false;

diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template
@@ -1124,6 +1124,10 @@ unseq_memtable_flush_check_interval_in_ms=30000
 # effectiveMode: restart
 tvlist_sort_algorithm=TIM
 
+# When point number in the working TVList exceeds this, it is sorted and handover in writable memtable
+# default 0 means it does not handover working tvlist
+tvlist_sort_threshold=0
+
 # When the average point number of timeseries in memtable exceeds this, the memtable is flushed to disk. The default threshold is 100000.
 # effectiveMode: restart
 # Datatype: int