Unaligned tvlist feat

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java

@@ -430,6 +430,9 @@ public class IoTDBConfig {
   /** The sort algorithm used in TVList */
   private TVListSortAlgorithm tvListSortAlgorithm = TVListSortAlgorithm.TIM;
 
+  /** the threshold when working TVList is sorted and handover in writable memtable */
+  private int tvListSortThreshold = 0;
+
   /** When average series point number reaches this, flush the memtable to disk */
   private int avgSeriesPointNumberThreshold = 100000;
 
@@ -2309,6 +2312,14 @@ public void setTvListSortAlgorithm(TVListSortAlgorithm tvListSortAlgorithm) {
     this.tvListSortAlgorithm = tvListSortAlgorithm;
   }
 
+  public int getTvListSortThreshold() {
+    return tvListSortThreshold;
+  }
+
+  public void setTVListSortThreshold(int tvListSortThreshold) {
+    this.tvListSortThreshold = tvListSortThreshold;
+  }
+
   public int getAvgSeriesPointNumberThreshold() {
     return avgSeriesPointNumberThreshold;
   }

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java

@@ -442,6 +442,14 @@ public void loadProperties(TrimProperties properties) throws BadNodeUrlException
                 .map(String::trim)
                 .orElse(conf.getTvListSortAlgorithm().toString())));
 
+    conf.setTVListSortThreshold(
+        Integer.parseInt(
+            Optional.ofNullable(
+                    properties.getProperty(
+                        "tvlist_sort_threshold", Integer.toString(conf.getTvListSortThreshold())))
+                .map(String::trim)
+                .orElse(Integer.toString(conf.getTvListSortThreshold()))));
+
     conf.setAvgSeriesPointNumberThreshold(
         Integer.parseInt(
             Optional.ofNullable(

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/FragmentInstanceContext.java

@@ -41,6 +41,7 @@
 import org.apache.iotdb.db.storageengine.dataregion.read.QueryDataSourceType;
 import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
+import org.apache.iotdb.db.utils.datastructure.TVList;
 import org.apache.iotdb.mpp.rpc.thrift.TFetchFragmentInstanceStatisticsResp;
 
 import org.apache.tsfile.file.metadata.IDeviceID;
@@ -649,6 +650,35 @@ public void releaseResourceWhenAllDriversAreClosed() {
     releaseResource();
   }
 
+  private void releaseTVListOwnedByQuery() {
+    for (TVList tvList : tvListSet) {
+      tvList.lockQueryList();
+      List<QueryContext> queryContextList = tvList.getQueryContextList();
+      try {
+        queryContextList.remove(this);
+        if (tvList.getOwnerQuery() == this) {
+          tvList.setOwnerQuery(null);
+          if (queryContextList.isEmpty()) {
+            LOGGER.debug(
+                "TVList {} is released by the query, FragmentInstance Id is {}",
+                tvList,
+                this.getId());
+            tvList.clear();
+            memoryReservationManager.releaseMemoryCumulatively(tvList.calculateRamSize());
+          } else {
+            LOGGER.debug(
+                "TVList {} is owned by another query, FragmentInstance Id is {}",
+                tvList,
+                ((FragmentInstanceContext) queryContextList.get(0)).getId());
+            tvList.setOwnerQuery(queryContextList.get(0));
+          }
+        }
+      } finally {
+        tvList.unlockQueryList();
+      }
+    }
+  }
+
   /**
    * All file paths used by this fragment instance must be cleared and thus the usage reference must
    * be decreased.
@@ -669,6 +699,9 @@ public synchronized void releaseResource() {
       unClosedFilePaths = null;
     }
 
+    // release TVList/AlignedTVList owned by current query
+    releaseTVListOwnedByQuery();
+
     dataRegion = null;
     globalTimeFilter = null;
     sharedQueryDataSource = null;

iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/fragment/QueryContext.java

@@ -26,15 +26,18 @@
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileID;
 import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource;
 import org.apache.iotdb.db.utils.ModificationUtils;
+import org.apache.iotdb.db.utils.datastructure.AlignedTVList;
 import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory;
 import org.apache.iotdb.db.utils.datastructure.PatternTreeMapFactory.ModsSerializer;
+import org.apache.iotdb.db.utils.datastructure.TVList;
 
 import org.apache.tsfile.file.metadata.IDeviceID;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -71,6 +74,9 @@ public class QueryContext {
 
   private final Set<TsFileID> nonExistentModFiles = new CopyOnWriteArraySet<>();
 
+  // accessed tvlists for the query
+  protected final Set<TVList> tvListSet = new HashSet<>();
+
   public QueryContext() {}
 
   public QueryContext(long queryId) {
@@ -206,4 +212,12 @@ public boolean isIgnoreAllNullRows() {
   public void setIgnoreAllNullRows(boolean ignoreAllNullRows) {
     this.ignoreAllNullRows = ignoreAllNullRows;
   }
+
+  public void addTVListToSet(Map<TVList, Integer> tvListMap) {
+    tvListSet.addAll(tvListMap.keySet());
+  }
+
+  public void addAlignedTVListToSet(Map<AlignedTVList, Integer> alignedTvListMap) {
+    tvListSet.addAll(alignedTvListMap.keySet());
+  }
 }