From d65c4f738ce0ffc57555d93cf8d75d0563479ef5 Mon Sep 17 00:00:00 2001 From: shizy Date: Sat, 7 Dec 2024 15:31:33 +0800 Subject: [PATCH] tvlist_sort_threshold config property --- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 11 +++++++++++ .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 8 ++++++++ .../dataregion/memtable/IWritableMemChunk.java | 4 ++-- .../dataregion/memtable/WritableMemChunk.java | 4 ++-- .../resources/conf/iotdb-system.properties.template | 4 ++++ 5 files changed, 27 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 942fe993c560b..507c6aaf5a201 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -430,6 +430,9 @@ public class IoTDBConfig { /** The sort algorithm used in TVList */ private TVListSortAlgorithm tvListSortAlgorithm = TVListSortAlgorithm.TIM; + /** the threshold when working TVList is sorted and handover in writable memtable */ + private int tvListSortThreshold = 0; + /** When average series point number reaches this, flush the memtable to disk */ private int avgSeriesPointNumberThreshold = 100000; @@ -2309,6 +2312,14 @@ public void setTvListSortAlgorithm(TVListSortAlgorithm tvListSortAlgorithm) { this.tvListSortAlgorithm = tvListSortAlgorithm; } + public int getTvListSortThreshold() { + return tvListSortThreshold; + } + + public void setTVListSortThreshold(int tvListSortThreshold) { + this.tvListSortThreshold = tvListSortThreshold; + } + public int getAvgSeriesPointNumberThreshold() { return avgSeriesPointNumberThreshold; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 37490ec97f249..e4ec90109b670 100755 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -442,6 +442,14 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException .map(String::trim) .orElse(conf.getTvListSortAlgorithm().toString()))); + conf.setTVListSortThreshold( + Integer.parseInt( + Optional.ofNullable( + properties.getProperty( + "tvlist_sort_threshold", Integer.toString(conf.getTvListSortThreshold()))) + .map(String::trim) + .orElse(Integer.toString(conf.getTvListSortThreshold())))); + conf.setAvgSeriesPointNumberThreshold( Integer.parseInt( Optional.ofNullable( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java index 8cc10502c7f95..b081e4b65dfef 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/IWritableMemChunk.java @@ -19,6 +19,7 @@ package org.apache.iotdb.db.storageengine.dataregion.memtable; import org.apache.iotdb.common.rpc.thrift.TSStatus; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; import org.apache.iotdb.db.utils.datastructure.TVList; @@ -31,8 +32,7 @@ import java.util.List; public interface IWritableMemChunk extends WALEntryValue { - // TODO: read from configuration file - int SORT_THRESHOLD = 5000; + int TVLIST_SORT_THRESHOLD = IoTDBDescriptor.getInstance().getConfig().getTvListSortThreshold(); boolean putLongWithFlushCheck(long t, long v); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java index eb044ba6a2099..209bb652aa03c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/memtable/WritableMemChunk.java @@ -129,7 +129,7 @@ public boolean writeWithFlushCheck(long insertTime, Object objectValue) { return true; } - if (list.rowCount() >= SORT_THRESHOLD) { + if (TVLIST_SORT_THRESHOLD > 0 && list.rowCount() >= TVLIST_SORT_THRESHOLD) { handoverTvList(); } return false; @@ -180,7 +180,7 @@ public boolean writeWithFlushCheck( return true; } - if (list.rowCount() >= SORT_THRESHOLD) { + if (TVLIST_SORT_THRESHOLD > 0 && list.rowCount() >= TVLIST_SORT_THRESHOLD) { handoverTvList(); } return false; diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template index cdcd5a8571a45..e31d369601be2 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties.template @@ -1124,6 +1124,10 @@ unseq_memtable_flush_check_interval_in_ms=30000 # effectiveMode: restart tvlist_sort_algorithm=TIM +# When point number in the working TVList exceeds this, it is sorted and handover in writable memtable +# default 0 means it does not handover working tvlist +tvlist_sort_threshold=0 + # When the average point number of timeseries in memtable exceeds this, the memtable is flushed to disk. The default threshold is 100000. # effectiveMode: restart # Datatype: int