From 8c52f5c54085070545e26aba26e02b917cb5fb77 Mon Sep 17 00:00:00 2001 From: LiuXuxin Date: Thu, 28 Mar 2024 22:46:06 +0800 Subject: [PATCH 01/32] enable wal compression remove metrics in mem table flush task, cache hash code in partial path, use gzip to compress wal batch update metrics --- .../org/apache/iotdb/db/conf/IoTDBConfig.java | 9 ++ .../apache/iotdb/db/conf/IoTDBDescriptor.java | 5 + .../plan/node/write/InsertRowNode.java | 2 + .../plan/node/write/InsertRowsNode.java | 5 + .../FragmentInstanceDispatcherImpl.java | 19 ++++ .../storageengine/dataregion/DataRegion.java | 31 +++++- .../dataregion/flush/MemTableFlushTask.java | 3 - .../dataregion/wal/io/CheckpointReader.java | 3 +- .../dataregion/wal/io/LogWriter.java | 36 ++++++ .../dataregion/wal/io/WALByteBufReader.java | 7 +- .../dataregion/wal/io/WALInputStream.java | 103 ++++++++++++++++++ .../dataregion/wal/io/WALReader.java | 6 +- .../dataregion/wal/io/WALWriter.java | 2 + .../dataregion/wal/utils/WALWriteUtils.java | 10 +- .../iotdb/commons/path/PartialPath.java | 16 ++- 15 files changed, 233 insertions(+), 24 deletions(-) create mode 100644 iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 9c20f6c32eca..d3d1a37e8977 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -95,6 +95,7 @@ public class IoTDBConfig { "([" + PATH_SEPARATOR + "])?" + NODE_NAME_MATCHER + "(" + PARTIAL_NODE_MATCHER + ")*"; public static final Pattern NODE_PATTERN = Pattern.compile(NODE_MATCHER); + boolean enableWALCompression = true; /** Whether to enable the mqtt service. */ private boolean enableMQTTService = false; @@ -3984,4 +3985,12 @@ public TDataNodeLocation generateLocalDataNodeLocation() { new TEndPoint(getInternalAddress(), getSchemaRegionConsensusPort())); return result; } + + public boolean isEnableWALCompression() { + return enableWALCompression; + } + + public void setEnableWALCompression(boolean enableWALCompression) { + this.enableWALCompression = enableWALCompression; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 81c60b9b8098..9dc53ba6a2e7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -417,6 +417,11 @@ public void loadProperties(Properties properties) throws BadNodeUrlException, IO "io_task_queue_size_for_flushing", Integer.toString(conf.getIoTaskQueueSizeForFlushing())))); + conf.setEnableWALCompression( + Boolean.parseBoolean( + properties.getProperty( + "enable_wal_compression", Boolean.toString(conf.isEnableWALCompression())))); + conf.setCompactionScheduleIntervalInMs( Long.parseLong( properties.getProperty( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java index bda7132fb7bf..c4c80e34fde3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java @@ -53,6 +53,7 @@ import java.util.Collections; import java.util.List; import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; public class InsertRowNode extends InsertNode implements WALEntryValue { @@ -68,6 +69,7 @@ public class InsertRowNode extends InsertNode implements WALEntryValue { private Object[] values; private boolean isNeedInferType = false; + public AtomicInteger insertCount; public InsertRowNode(PlanNodeId id) { super(id); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java index 49fb7fac6318..4684ac8e76a5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java @@ -47,6 +47,8 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; public class InsertRowsNode extends InsertNode implements WALEntryValue { @@ -65,6 +67,9 @@ public class InsertRowsNode extends InsertNode implements WALEntryValue { /** The {@link InsertRowNode} list */ private List insertRowNodeList; + public AtomicInteger insertCount = new AtomicInteger(0); + public AtomicLong[] metrics; + public InsertRowsNode(PlanNodeId id) { super(id); insertRowNodeList = new ArrayList<>(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java index 7fe941a6f838..5183e43c8c0a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java @@ -40,6 +40,7 @@ import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.utils.SetThreadName; import org.apache.iotdb.mpp.rpc.thrift.TFragmentInstance; import org.apache.iotdb.mpp.rpc.thrift.TPlanNode; @@ -62,6 +63,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import static com.google.common.util.concurrent.Futures.immediateFuture; import static org.apache.iotdb.db.queryengine.metric.QueryExecutionMetricSet.DISPATCH_READ; @@ -433,6 +435,13 @@ private void dispatchLocally(FragmentInstance instance) throws FragmentInstanceD break; case WRITE: PlanNode planNode = instance.getFragment().getPlanNodeTree(); + if (planNode instanceof InsertRowsNode) { + InsertRowsNode insertRowsNode = (InsertRowsNode) planNode; + insertRowsNode.metrics = new AtomicLong[4]; + for (int i = 0; i < 4; i++) { + insertRowsNode.metrics[i] = new AtomicLong(0); + } + } RegionWriteExecutor writeExecutor = new RegionWriteExecutor(); RegionExecutionResult writeResult = writeExecutor.execute(groupId, planNode); if (!writeResult.isAccepted()) { @@ -452,6 +461,16 @@ private void dispatchLocally(FragmentInstance instance) throws FragmentInstanceD } else { // some expected and accepted status except SUCCESS_STATUS need to be returned TSStatus status = writeResult.getStatus(); + if (planNode instanceof InsertRowsNode) { + InsertRowsNode insertRowsNode = (InsertRowsNode) planNode; + PERFORMANCE_OVERVIEW_METRICS.recordCreateMemtableBlockCost( + insertRowsNode.metrics[0].get()); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleMemoryBlockCost( + insertRowsNode.metrics[1].get()); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleWalCost(insertRowsNode.metrics[2].get()); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleMemTableCost( + insertRowsNode.metrics[3].get()); + } if (status != null && status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { throw new FragmentInstanceDispatchException(status); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 5a58b3e1dbfc..6a09a491f4cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -29,6 +29,8 @@ import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; +import org.apache.iotdb.commons.service.metric.enums.Metric; +import org.apache.iotdb.commons.service.metric.enums.Tag; import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.commons.utils.TimePartitionUtils; @@ -110,6 +112,7 @@ import org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeSpaceQuotaManager; import org.apache.iotdb.db.tools.settle.TsFileAndModSettleTool; import org.apache.iotdb.db.utils.DateTimeUtils; +import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -1186,6 +1189,7 @@ private void insertToTsFileProcessors( if (insertRowNode.allMeasurementFailed()) { continue; } + insertRowNode.insertCount = insertRowsNode.insertCount; TsFileProcessor tsFileProcessor = getOrCreateTsFileProcessor(timePartitionIds[i], areSequence[i]); if (tsFileProcessor == null) { @@ -1226,10 +1230,29 @@ private void insertToTsFileProcessors( } } - PERFORMANCE_OVERVIEW_METRICS.recordCreateMemtableBlockCost(costsForMetrics[0]); - PERFORMANCE_OVERVIEW_METRICS.recordScheduleMemoryBlockCost(costsForMetrics[1]); - PERFORMANCE_OVERVIEW_METRICS.recordScheduleWalCost(costsForMetrics[2]); - PERFORMANCE_OVERVIEW_METRICS.recordScheduleMemTableCost(costsForMetrics[3]); + MetricService.getInstance() + .count( + insertRowsNode.insertCount.get(), + Metric.QUANTITY.toString(), + MetricLevel.CORE, + Tag.NAME.toString(), + Metric.POINTS_IN.toString(), + Tag.DATABASE.toString(), + databaseName, + Tag.REGION.toString(), + dataRegionId); + + if (insertRowsNode.metrics != null) { + insertRowsNode.metrics[0].addAndGet(costsForMetrics[0]); + insertRowsNode.metrics[1].addAndGet(costsForMetrics[1]); + insertRowsNode.metrics[2].addAndGet(costsForMetrics[2]); + insertRowsNode.metrics[3].addAndGet(costsForMetrics[3]); + } else { + PERFORMANCE_OVERVIEW_METRICS.recordCreateMemtableBlockCost(costsForMetrics[0]); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleMemoryBlockCost(costsForMetrics[1]); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleWalCost(costsForMetrics[2]); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleMemTableCost(costsForMetrics[3]); + } if (CommonDescriptor.getInstance().getConfig().isLastCacheEnable()) { if ((config.getDataRegionConsensusProtocolClass().equals(ConsensusFactory.IOT_CONSENSUS) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java index 45a398b96bb5..948273fb84f9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java @@ -156,7 +156,6 @@ public void syncFlushMemTable() throws ExecutionException, InterruptedException series.sortTvListForFlush(); long subTaskTime = System.currentTimeMillis() - startTime; sortTime += subTaskTime; - WRITING_METRICS.recordFlushSubTaskCost(WritingMetrics.SORT_TASK, subTaskTime); encodingTaskQueue.put(series); } @@ -258,7 +257,6 @@ public void run() { Thread.currentThread().interrupt(); } long subTaskTime = System.currentTimeMillis() - starTime; - WRITING_METRICS.recordFlushSubTaskCost(WritingMetrics.ENCODING_TASK, subTaskTime); memSerializeTime += subTaskTime; } } @@ -344,7 +342,6 @@ public static void recordFlushPointsMetricInternal( } long subTaskTime = System.currentTimeMillis() - starTime; ioTime += subTaskTime; - WRITING_METRICS.recordFlushSubTaskCost(WritingMetrics.IO_TASK, subTaskTime); } LOGGER.debug( "flushing a memtable to file {} in database {}, io cost {}ms", diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointReader.java index 5d2bad0a8740..081b3ed4a4fd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointReader.java @@ -27,7 +27,6 @@ import java.io.BufferedInputStream; import java.io.DataInputStream; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -48,7 +47,7 @@ public CheckpointReader(File logFile) { private void init() { checkpoints = new ArrayList<>(); try (DataInputStream logStream = - new DataInputStream(new BufferedInputStream(new FileInputStream(logFile)))) { + new DataInputStream(new BufferedInputStream(new WALInputStream(logFile)))) { maxMemTableId = logStream.readLong(); while (logStream.available() > 0) { Checkpoint checkpoint = Checkpoint.deserialize(logStream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index 68f4deae3189..c3fe218fb40e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -19,8 +19,11 @@ package org.apache.iotdb.db.storageengine.dataregion.wal.io; +import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry; import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.Checkpoint; +import org.apache.iotdb.tsfile.compress.ICompressor; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,18 +47,51 @@ public abstract class LogWriter implements ILogWriter { protected final FileOutputStream logStream; protected final FileChannel logChannel; protected long size; + protected boolean isEndFile = false; + private final ByteBuffer headerBuffer = ByteBuffer.allocate(Integer.BYTES * 2 + 1); + private final ICompressor compressor = ICompressor.getCompressor(CompressionType.GZIP); + private final ByteBuffer compressedByteBuffer; protected LogWriter(File logFile) throws FileNotFoundException { this.logFile = logFile; this.logStream = new FileOutputStream(logFile, true); this.logChannel = this.logStream.getChannel(); + if (IoTDBDescriptor.getInstance().getConfig().isEnableWALCompression()) { + compressedByteBuffer = + ByteBuffer.allocate( + compressor.getMaxBytesForCompression( + IoTDBDescriptor.getInstance().getConfig().getWalBufferSize())); + } else { + compressedByteBuffer = null; + } } @Override public void write(ByteBuffer buffer) throws IOException { + int bufferSize = buffer.position(); size += buffer.position(); buffer.flip(); + boolean compressed = false; + int uncompressedSize = bufferSize; + if (!isEndFile && IoTDBDescriptor.getInstance().getConfig().isEnableWALCompression() + /* && bufferSize > 1024 * 512 Do not compress buffer that is less than 512KB */ ) { + compressedByteBuffer.clear(); + compressor.compress(buffer, compressedByteBuffer); + buffer = compressedByteBuffer; + bufferSize = buffer.position(); + buffer.flip(); + compressed = true; + } + size += bufferSize; + headerBuffer.clear(); + headerBuffer.putInt(bufferSize); + headerBuffer.put((byte) (compressed ? 1 : 0)); try { + if (compressed) { + headerBuffer.putInt(uncompressedSize); + } + headerBuffer.flip(); + logChannel.write(headerBuffer); logChannel.write(buffer); } catch (ClosedChannelException e) { logger.warn("Cannot write to {}", logFile, e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java index f101eaf3647e..ad3b7479de95 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java @@ -22,6 +22,7 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry; import java.io.Closeable; +import java.io.DataInputStream; import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; @@ -37,6 +38,7 @@ public class WALByteBufReader implements Closeable { private final File logFile; private final FileChannel channel; private final WALMetaData metaData; + private final DataInputStream logStream; private final Iterator sizeIterator; public WALByteBufReader(File logFile) throws IOException { @@ -46,6 +48,7 @@ public WALByteBufReader(File logFile) throws IOException { public WALByteBufReader(File logFile, FileChannel channel) throws IOException { this.logFile = logFile; this.channel = channel; + this.logStream = new DataInputStream(new WALInputStream(logFile)); this.metaData = WALMetaData.readFromWALFile(logFile, channel); this.sizeIterator = metaData.getBuffersSize().iterator(); channel.position(0); @@ -64,8 +67,8 @@ public boolean hasNext() { public ByteBuffer next() throws IOException { int size = sizeIterator.next(); ByteBuffer buffer = ByteBuffer.allocate(size); - channel.read(buffer); - buffer.clear(); + logStream.readFully(buffer.array(), 0, size); + buffer.flip(); return buffer; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java new file mode 100644 index 000000000000..8e742b3cb1b8 --- /dev/null +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java @@ -0,0 +1,103 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.storageengine.dataregion.wal.io; + +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.tsfile.compress.IUnCompressor; +import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; +import java.nio.channels.FileChannel; +import java.util.Objects; + +public class WALInputStream extends InputStream implements AutoCloseable { + + private static final Logger logger = LoggerFactory.getLogger(WALInputStream.class); + private final FileChannel channel; + private final ByteBuffer headerBuffer = ByteBuffer.allocate(Integer.BYTES + 1); + private final ByteBuffer compressedHeader = ByteBuffer.allocate(Integer.BYTES); + private ByteBuffer dataBuffer = + ByteBuffer.allocate( + IoTDBDescriptor.getInstance().getConfig().getWalBufferSize()); // uncompressed data buffer + + public WALInputStream(File logFile) throws IOException { + channel = FileChannel.open(logFile.toPath()); + } + + @Override + public int read() throws IOException { + if (Objects.isNull(dataBuffer) || dataBuffer.position() == dataBuffer.limit()) { + loadNextSegment(); + } + return dataBuffer.get() & 0xFF; + } + + @Override + public void close() throws IOException { + channel.close(); + dataBuffer = null; + } + + @Override + public int available() throws IOException { + return (int) (channel.size() - channel.position()); + } + + private void loadNextSegment() throws IOException { + headerBuffer.clear(); + if (channel.read(headerBuffer) != Integer.BYTES + 1) { + throw new IOException("Unexpected end of file"); + } + headerBuffer.flip(); + int dataSize = headerBuffer.getInt(); + boolean isCompressed = headerBuffer.get() == 1; + if (isCompressed) { + compressedHeader.clear(); + if (channel.read(compressedHeader) != Integer.BYTES) { + throw new IOException("Unexpected end of file"); + } + compressedHeader.flip(); + int uncompressedSize = compressedHeader.getInt(); + if (uncompressedSize > dataBuffer.capacity()) { + // enlarge buffer + dataBuffer = ByteBuffer.allocateDirect(uncompressedSize); + } + ByteBuffer compressedData = ByteBuffer.allocateDirect(dataSize); + if (channel.read(compressedData) != dataSize) { + throw new IOException("Unexpected end of file"); + } + compressedData.flip(); + IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.LZ4); + dataBuffer.clear(); + unCompressor.uncompress(compressedData, dataBuffer); + } else { + dataBuffer = ByteBuffer.allocateDirect(dataSize); + if (channel.read(dataBuffer) != dataSize) { + throw new IOException("Unexpected end of file"); + } + } + dataBuffer.flip(); + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALReader.java index ee50c73df972..475ea2b0b2d8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALReader.java @@ -26,12 +26,10 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedInputStream; import java.io.Closeable; import java.io.DataInputStream; import java.io.File; import java.io.IOException; -import java.nio.file.Files; import java.util.Iterator; import java.util.NoSuchElementException; @@ -57,9 +55,7 @@ public WALReader(File logFile) throws IOException { public WALReader(File logFile, boolean fileMayCorrupt) throws IOException { this.logFile = logFile; this.fileMayCorrupt = fileMayCorrupt; - this.logStream = - new DataInputStream( - new BufferedInputStream(Files.newInputStream(logFile.toPath()), STREAM_BUFFER_SIZE)); + this.logStream = new DataInputStream(new WALInputStream(logFile)); } /** Like {@link Iterator#hasNext()}. */ diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java index 425fc676fad8..20ae99754505 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java @@ -59,6 +59,7 @@ public void updateMetaData(WALMetaData metaData) { } private void endFile() throws IOException { + this.isEndFile = true; WALSignalEntry endMarker = new WALSignalEntry(WALEntryType.WAL_FILE_INFO_END_MARKER); int metaDataSize = metaData.serializedSize(); ByteBuffer buffer = @@ -72,6 +73,7 @@ private void endFile() throws IOException { // add magic string buffer.put(MAGIC_STRING.getBytes()); write(buffer); + this.isEndFile = false; } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALWriteUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALWriteUtils.java index e6c0eb02722a..64aba4872757 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALWriteUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALWriteUtils.java @@ -127,10 +127,12 @@ public static int write(String s, IWALByteBufferView buffer) { return write(NO_BYTE_TO_READ, buffer); } int len = 0; - byte[] bytes = s.getBytes(); - len += write(bytes.length, buffer); - buffer.put(bytes); - len += bytes.length; + len += write(s.length(), buffer); + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + buffer.put((byte) c); // ascii only + } + len += s.length(); return len; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java index 3327f4c9bbea..5df64ecfce17 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java @@ -58,6 +58,8 @@ public class PartialPath extends Path implements Comparable, Cloneable { private static final PartialPath ALL_MATCH_PATTERN = new PartialPath(new String[] {"root", "**"}); protected String[] nodes; + protected int hashCache; + protected boolean cacheHashCache = false; public PartialPath() {} @@ -717,11 +719,17 @@ public boolean equals(String obj) { @Override public int hashCode() { - int h = 0; - for (String node : nodes) { - h += 31 * h + node.hashCode(); + if (cacheHashCache) { + return hashCache; + } else { + int h = 0; + for (String node : nodes) { + h += 31 * h + node.hashCode(); + } + hashCache = h; + cacheHashCache = true; + return h; } - return h; } @Override From 693ffb781de0a44adb73b51301a253490306ceb6 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Sat, 30 Mar 2024 17:36:53 +0800 Subject: [PATCH 02/32] fix bug --- .../apache/iotdb/consensus/iot/IoTConsensusServerImpl.java | 2 +- .../iotdb/db/storageengine/dataregion/wal/io/LogWriter.java | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java index 4ac574d282af..b7404492df25 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java @@ -811,7 +811,7 @@ public long getLogEntriesFromQueue() { } public boolean needBlockWrite() { - return consensusReqReader.getTotalSize() > config.getReplication().getWalThrottleThreshold(); + return false; } public boolean unblockWrite() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index c3fe218fb40e..4e5d6239b32c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -57,6 +57,7 @@ protected LogWriter(File logFile) throws FileNotFoundException { this.logStream = new FileOutputStream(logFile, true); this.logChannel = this.logStream.getChannel(); if (IoTDBDescriptor.getInstance().getConfig().isEnableWALCompression()) { + logger.info("Enable WAL compression with gzip"); compressedByteBuffer = ByteBuffer.allocate( compressor.getMaxBytesForCompression( @@ -69,7 +70,6 @@ protected LogWriter(File logFile) throws FileNotFoundException { @Override public void write(ByteBuffer buffer) throws IOException { int bufferSize = buffer.position(); - size += buffer.position(); buffer.flip(); boolean compressed = false; int uncompressedSize = bufferSize; @@ -86,9 +86,12 @@ public void write(ByteBuffer buffer) throws IOException { headerBuffer.clear(); headerBuffer.putInt(bufferSize); headerBuffer.put((byte) (compressed ? 1 : 0)); + size += bufferSize; + size += 5; try { if (compressed) { headerBuffer.putInt(uncompressedSize); + size += 4; } headerBuffer.flip(); logChannel.write(headerBuffer); From e2e6bc11f6ea5dfb79eea498ec225ac487d1134e Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Tue, 7 May 2024 16:04:25 +0800 Subject: [PATCH 03/32] fix compilation problem --- .../apache/iotdb/consensus/iot/IoTConsensusServerImpl.java | 2 +- .../iotdb/db/storageengine/dataregion/wal/io/LogWriter.java | 5 ++--- .../db/storageengine/dataregion/wal/io/WALInputStream.java | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java index b7404492df25..4ac574d282af 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java @@ -811,7 +811,7 @@ public long getLogEntriesFromQueue() { } public boolean needBlockWrite() { - return false; + return consensusReqReader.getTotalSize() > config.getReplication().getWalThrottleThreshold(); } public boolean unblockWrite() { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index 4e5d6239b32c..95bf594dcc3d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -22,9 +22,9 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry; import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.Checkpoint; -import org.apache.iotdb.tsfile.compress.ICompressor; -import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.compress.ICompressor; +import org.apache.tsfile.file.metadata.enums.CompressionType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,7 +57,6 @@ protected LogWriter(File logFile) throws FileNotFoundException { this.logStream = new FileOutputStream(logFile, true); this.logChannel = this.logStream.getChannel(); if (IoTDBDescriptor.getInstance().getConfig().isEnableWALCompression()) { - logger.info("Enable WAL compression with gzip"); compressedByteBuffer = ByteBuffer.allocate( compressor.getMaxBytesForCompression( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java index 8e742b3cb1b8..f75119590fc3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java @@ -19,9 +19,9 @@ package org.apache.iotdb.db.storageengine.dataregion.wal.io; import org.apache.iotdb.db.conf.IoTDBDescriptor; -import org.apache.iotdb.tsfile.compress.IUnCompressor; -import org.apache.iotdb.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.compress.IUnCompressor; +import org.apache.tsfile.file.metadata.enums.CompressionType; import org.slf4j.Logger; import org.slf4j.LoggerFactory; From 896a357df3ac3486a7159d50d6aa6272b153ea10 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Tue, 7 May 2024 16:24:53 +0800 Subject: [PATCH 04/32] remove useless code --- .../plan/node/write/InsertRowNode.java | 2 -- .../plan/node/write/InsertRowsNode.java | 5 --- .../FragmentInstanceDispatcherImpl.java | 19 ------------ .../storageengine/dataregion/DataRegion.java | 31 +++---------------- .../dataregion/wal/io/LogWriter.java | 12 ++++--- 5 files changed, 11 insertions(+), 58 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java index c4c80e34fde3..bda7132fb7bf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowNode.java @@ -53,7 +53,6 @@ import java.util.Collections; import java.util.List; import java.util.Objects; -import java.util.concurrent.atomic.AtomicInteger; public class InsertRowNode extends InsertNode implements WALEntryValue { @@ -69,7 +68,6 @@ public class InsertRowNode extends InsertNode implements WALEntryValue { private Object[] values; private boolean isNeedInferType = false; - public AtomicInteger insertCount; public InsertRowNode(PlanNodeId id) { super(id); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java index 4684ac8e76a5..49fb7fac6318 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/planner/plan/node/write/InsertRowsNode.java @@ -47,8 +47,6 @@ import java.util.List; import java.util.Map; import java.util.Objects; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; public class InsertRowsNode extends InsertNode implements WALEntryValue { @@ -67,9 +65,6 @@ public class InsertRowsNode extends InsertNode implements WALEntryValue { /** The {@link InsertRowNode} list */ private List insertRowNodeList; - public AtomicInteger insertCount = new AtomicInteger(0); - public AtomicLong[] metrics; - public InsertRowsNode(PlanNodeId id) { super(id); insertRowNodeList = new ArrayList<>(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java index 5183e43c8c0a..7fe941a6f838 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/scheduler/FragmentInstanceDispatcherImpl.java @@ -40,7 +40,6 @@ import org.apache.iotdb.db.queryengine.plan.analyze.QueryType; import org.apache.iotdb.db.queryengine.plan.planner.plan.FragmentInstance; import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNode; -import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowsNode; import org.apache.iotdb.db.utils.SetThreadName; import org.apache.iotdb.mpp.rpc.thrift.TFragmentInstance; import org.apache.iotdb.mpp.rpc.thrift.TPlanNode; @@ -63,7 +62,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicLong; import static com.google.common.util.concurrent.Futures.immediateFuture; import static org.apache.iotdb.db.queryengine.metric.QueryExecutionMetricSet.DISPATCH_READ; @@ -435,13 +433,6 @@ private void dispatchLocally(FragmentInstance instance) throws FragmentInstanceD break; case WRITE: PlanNode planNode = instance.getFragment().getPlanNodeTree(); - if (planNode instanceof InsertRowsNode) { - InsertRowsNode insertRowsNode = (InsertRowsNode) planNode; - insertRowsNode.metrics = new AtomicLong[4]; - for (int i = 0; i < 4; i++) { - insertRowsNode.metrics[i] = new AtomicLong(0); - } - } RegionWriteExecutor writeExecutor = new RegionWriteExecutor(); RegionExecutionResult writeResult = writeExecutor.execute(groupId, planNode); if (!writeResult.isAccepted()) { @@ -461,16 +452,6 @@ private void dispatchLocally(FragmentInstance instance) throws FragmentInstanceD } else { // some expected and accepted status except SUCCESS_STATUS need to be returned TSStatus status = writeResult.getStatus(); - if (planNode instanceof InsertRowsNode) { - InsertRowsNode insertRowsNode = (InsertRowsNode) planNode; - PERFORMANCE_OVERVIEW_METRICS.recordCreateMemtableBlockCost( - insertRowsNode.metrics[0].get()); - PERFORMANCE_OVERVIEW_METRICS.recordScheduleMemoryBlockCost( - insertRowsNode.metrics[1].get()); - PERFORMANCE_OVERVIEW_METRICS.recordScheduleWalCost(insertRowsNode.metrics[2].get()); - PERFORMANCE_OVERVIEW_METRICS.recordScheduleMemTableCost( - insertRowsNode.metrics[3].get()); - } if (status != null && status.getCode() != TSStatusCode.SUCCESS_STATUS.getStatusCode()) { throw new FragmentInstanceDispatchException(status); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 6a09a491f4cc..5a58b3e1dbfc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -29,8 +29,6 @@ import org.apache.iotdb.commons.schema.SchemaConstant; import org.apache.iotdb.commons.service.metric.MetricService; import org.apache.iotdb.commons.service.metric.PerformanceOverviewMetrics; -import org.apache.iotdb.commons.service.metric.enums.Metric; -import org.apache.iotdb.commons.service.metric.enums.Tag; import org.apache.iotdb.commons.utils.CommonDateTimeUtils; import org.apache.iotdb.commons.utils.TestOnly; import org.apache.iotdb.commons.utils.TimePartitionUtils; @@ -112,7 +110,6 @@ import org.apache.iotdb.db.storageengine.rescon.quotas.DataNodeSpaceQuotaManager; import org.apache.iotdb.db.tools.settle.TsFileAndModSettleTool; import org.apache.iotdb.db.utils.DateTimeUtils; -import org.apache.iotdb.metrics.utils.MetricLevel; import org.apache.iotdb.rpc.RpcUtils; import org.apache.iotdb.rpc.TSStatusCode; @@ -1189,7 +1186,6 @@ private void insertToTsFileProcessors( if (insertRowNode.allMeasurementFailed()) { continue; } - insertRowNode.insertCount = insertRowsNode.insertCount; TsFileProcessor tsFileProcessor = getOrCreateTsFileProcessor(timePartitionIds[i], areSequence[i]); if (tsFileProcessor == null) { @@ -1230,29 +1226,10 @@ private void insertToTsFileProcessors( } } - MetricService.getInstance() - .count( - insertRowsNode.insertCount.get(), - Metric.QUANTITY.toString(), - MetricLevel.CORE, - Tag.NAME.toString(), - Metric.POINTS_IN.toString(), - Tag.DATABASE.toString(), - databaseName, - Tag.REGION.toString(), - dataRegionId); - - if (insertRowsNode.metrics != null) { - insertRowsNode.metrics[0].addAndGet(costsForMetrics[0]); - insertRowsNode.metrics[1].addAndGet(costsForMetrics[1]); - insertRowsNode.metrics[2].addAndGet(costsForMetrics[2]); - insertRowsNode.metrics[3].addAndGet(costsForMetrics[3]); - } else { - PERFORMANCE_OVERVIEW_METRICS.recordCreateMemtableBlockCost(costsForMetrics[0]); - PERFORMANCE_OVERVIEW_METRICS.recordScheduleMemoryBlockCost(costsForMetrics[1]); - PERFORMANCE_OVERVIEW_METRICS.recordScheduleWalCost(costsForMetrics[2]); - PERFORMANCE_OVERVIEW_METRICS.recordScheduleMemTableCost(costsForMetrics[3]); - } + PERFORMANCE_OVERVIEW_METRICS.recordCreateMemtableBlockCost(costsForMetrics[0]); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleMemoryBlockCost(costsForMetrics[1]); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleWalCost(costsForMetrics[2]); + PERFORMANCE_OVERVIEW_METRICS.recordScheduleMemTableCost(costsForMetrics[3]); if (CommonDescriptor.getInstance().getConfig().isLastCacheEnable()) { if ((config.getDataRegionConsensusProtocolClass().equals(ConsensusFactory.IOT_CONSENSUS) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index 95bf594dcc3d..a3a080637887 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -49,6 +49,7 @@ public abstract class LogWriter implements ILogWriter { protected long size; protected boolean isEndFile = false; private final ByteBuffer headerBuffer = ByteBuffer.allocate(Integer.BYTES * 2 + 1); + private static final CompressionType compressionType = CompressionType.GZIP; private final ICompressor compressor = ICompressor.getCompressor(CompressionType.GZIP); private final ByteBuffer compressedByteBuffer; @@ -83,15 +84,16 @@ public void write(ByteBuffer buffer) throws IOException { } size += bufferSize; headerBuffer.clear(); + headerBuffer.put( + compressed ? compressionType.serialize() : CompressionType.UNCOMPRESSED.serialize()); + if (compressed) { + headerBuffer.putInt(uncompressedSize); + size += 4; + } headerBuffer.putInt(bufferSize); - headerBuffer.put((byte) (compressed ? 1 : 0)); size += bufferSize; size += 5; try { - if (compressed) { - headerBuffer.putInt(uncompressedSize); - size += 4; - } headerBuffer.flip(); logChannel.write(headerBuffer); logChannel.write(buffer); From ee3a64f720c6198c46faddb7e9916ef2a3e0ce55 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Tue, 7 May 2024 16:28:51 +0800 Subject: [PATCH 05/32] recover some code --- .../dataregion/flush/MemTableFlushTask.java | 3 +++ .../dataregion/wal/utils/WALWriteUtils.java | 10 ++++------ .../apache/iotdb/commons/path/PartialPath.java | 16 ++++------------ 3 files changed, 11 insertions(+), 18 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java index 948273fb84f9..45a398b96bb5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/flush/MemTableFlushTask.java @@ -156,6 +156,7 @@ public void syncFlushMemTable() throws ExecutionException, InterruptedException series.sortTvListForFlush(); long subTaskTime = System.currentTimeMillis() - startTime; sortTime += subTaskTime; + WRITING_METRICS.recordFlushSubTaskCost(WritingMetrics.SORT_TASK, subTaskTime); encodingTaskQueue.put(series); } @@ -257,6 +258,7 @@ public void run() { Thread.currentThread().interrupt(); } long subTaskTime = System.currentTimeMillis() - starTime; + WRITING_METRICS.recordFlushSubTaskCost(WritingMetrics.ENCODING_TASK, subTaskTime); memSerializeTime += subTaskTime; } } @@ -342,6 +344,7 @@ public static void recordFlushPointsMetricInternal( } long subTaskTime = System.currentTimeMillis() - starTime; ioTime += subTaskTime; + WRITING_METRICS.recordFlushSubTaskCost(WritingMetrics.IO_TASK, subTaskTime); } LOGGER.debug( "flushing a memtable to file {} in database {}, io cost {}ms", diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALWriteUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALWriteUtils.java index 64aba4872757..e6c0eb02722a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALWriteUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALWriteUtils.java @@ -127,12 +127,10 @@ public static int write(String s, IWALByteBufferView buffer) { return write(NO_BYTE_TO_READ, buffer); } int len = 0; - len += write(s.length(), buffer); - for (int i = 0; i < s.length(); i++) { - char c = s.charAt(i); - buffer.put((byte) c); // ascii only - } - len += s.length(); + byte[] bytes = s.getBytes(); + len += write(bytes.length, buffer); + buffer.put(bytes); + len += bytes.length; return len; } diff --git a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java index 5df64ecfce17..3327f4c9bbea 100644 --- a/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java +++ b/iotdb-core/node-commons/src/main/java/org/apache/iotdb/commons/path/PartialPath.java @@ -58,8 +58,6 @@ public class PartialPath extends Path implements Comparable, Cloneable { private static final PartialPath ALL_MATCH_PATTERN = new PartialPath(new String[] {"root", "**"}); protected String[] nodes; - protected int hashCache; - protected boolean cacheHashCache = false; public PartialPath() {} @@ -719,17 +717,11 @@ public boolean equals(String obj) { @Override public int hashCode() { - if (cacheHashCache) { - return hashCache; - } else { - int h = 0; - for (String node : nodes) { - h += 31 * h + node.hashCode(); - } - hashCache = h; - cacheHashCache = true; - return h; + int h = 0; + for (String node : nodes) { + h += 31 * h + node.hashCode(); } + return h; } @Override From 1316653b5bff2225510eb8f34678b8a82c21ad1d Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Tue, 7 May 2024 16:34:06 +0800 Subject: [PATCH 06/32] support compression type in WAL Compress Header --- .../dataregion/wal/io/LogWriter.java | 2 +- .../dataregion/wal/io/WALInputStream.java | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index a3a080637887..94dfbc489c4e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -86,11 +86,11 @@ public void write(ByteBuffer buffer) throws IOException { headerBuffer.clear(); headerBuffer.put( compressed ? compressionType.serialize() : CompressionType.UNCOMPRESSED.serialize()); + headerBuffer.putInt(bufferSize); if (compressed) { headerBuffer.putInt(uncompressedSize); size += 4; } - headerBuffer.putInt(bufferSize); size += bufferSize; size += 5; try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java index f75119590fc3..d7488ad99f42 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java @@ -70,10 +70,11 @@ private void loadNextSegment() throws IOException { if (channel.read(headerBuffer) != Integer.BYTES + 1) { throw new IOException("Unexpected end of file"); } + // compressionType originalSize compressedSize headerBuffer.flip(); - int dataSize = headerBuffer.getInt(); - boolean isCompressed = headerBuffer.get() == 1; - if (isCompressed) { + CompressionType compressionType = CompressionType.deserialize(headerBuffer.get()); + int dataBufferSize = headerBuffer.getInt(); + if (compressionType != CompressionType.UNCOMPRESSED) { compressedHeader.clear(); if (channel.read(compressedHeader) != Integer.BYTES) { throw new IOException("Unexpected end of file"); @@ -84,17 +85,17 @@ private void loadNextSegment() throws IOException { // enlarge buffer dataBuffer = ByteBuffer.allocateDirect(uncompressedSize); } - ByteBuffer compressedData = ByteBuffer.allocateDirect(dataSize); - if (channel.read(compressedData) != dataSize) { + ByteBuffer compressedData = ByteBuffer.allocate(dataBufferSize); + if (channel.read(compressedData) != dataBufferSize) { throw new IOException("Unexpected end of file"); } compressedData.flip(); - IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.LZ4); + IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.GZIP); dataBuffer.clear(); unCompressor.uncompress(compressedData, dataBuffer); } else { - dataBuffer = ByteBuffer.allocateDirect(dataSize); - if (channel.read(dataBuffer) != dataSize) { + dataBuffer = ByteBuffer.allocateDirect(dataBufferSize); + if (channel.read(dataBuffer) != dataBufferSize) { throw new IOException("Unexpected end of file"); } } From 2728057d9c1352650397cdbe695043d6a1626a17 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Sat, 11 May 2024 16:05:34 +0800 Subject: [PATCH 07/32] support multi version WAL --- .../dataregion/wal/io/LogWriter.java | 10 +++- .../dataregion/wal/io/WALInputStream.java | 56 +++++++++++++++++++ .../dataregion/wal/io/WALMetaData.java | 9 +-- .../dataregion/wal/io/WALWriter.java | 3 +- .../wal/recover/WALRecoverWriter.java | 4 +- 5 files changed, 73 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index 94dfbc489c4e..44ef2f4e1965 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -52,6 +52,7 @@ public abstract class LogWriter implements ILogWriter { private static final CompressionType compressionType = CompressionType.GZIP; private final ICompressor compressor = ICompressor.getCompressor(CompressionType.GZIP); private final ByteBuffer compressedByteBuffer; + private static final long MIN_COMPRESS_SIZE = 1024 * 512; protected LogWriter(File logFile) throws FileNotFoundException { this.logFile = logFile; @@ -73,8 +74,9 @@ public void write(ByteBuffer buffer) throws IOException { buffer.flip(); boolean compressed = false; int uncompressedSize = bufferSize; - if (!isEndFile && IoTDBDescriptor.getInstance().getConfig().isEnableWALCompression() - /* && bufferSize > 1024 * 512 Do not compress buffer that is less than 512KB */ ) { + if (!isEndFile + && IoTDBDescriptor.getInstance().getConfig().isEnableWALCompression() + && bufferSize > MIN_COMPRESS_SIZE /* Do not compress buffer that is less than 512KB */) { compressedByteBuffer.clear(); compressor.compress(buffer, compressedByteBuffer); buffer = compressedByteBuffer; @@ -83,6 +85,10 @@ public void write(ByteBuffer buffer) throws IOException { compressed = true; } size += bufferSize; + /* + Header structure: + [CompressionType(1 byte)][dataBufferSize(4 bytes)][uncompressedSize(4 bytes)] + */ headerBuffer.clear(); headerBuffer.put( compressed ? compressionType.serialize() : CompressionType.UNCOMPRESSED.serialize()); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java index d7488ad99f42..33287a511a32 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java @@ -42,8 +42,31 @@ public class WALInputStream extends InputStream implements AutoCloseable { ByteBuffer.allocate( IoTDBDescriptor.getInstance().getConfig().getWalBufferSize()); // uncompressed data buffer + enum FileVersion { + V1, + V2, + UNKNOWN + }; + + FileVersion version; + public WALInputStream(File logFile) throws IOException { channel = FileChannel.open(logFile.toPath()); + analyzeFileVersion(); + } + + private void analyzeFileVersion() throws IOException { + ByteBuffer magicStringBytes = ByteBuffer.allocate(WALWriter.MAGIC_STRING_BYTES); + channel.read(magicStringBytes, channel.size() - WALWriter.MAGIC_STRING_BYTES); + magicStringBytes.flip(); + String magicString = new String(magicStringBytes.array()); + if (magicString.equals(WALWriter.MAGIC_STRING)) { + version = FileVersion.V2; + } else if (magicString.startsWith(WALWriter.MAGIC_STRING_V1)) { + version = FileVersion.V1; + } else { + version = FileVersion.UNKNOWN; + } } @Override @@ -66,6 +89,22 @@ public int available() throws IOException { } private void loadNextSegment() throws IOException { + if (version == FileVersion.V2) { + loadNextSegmentV2(); + } else if (version == FileVersion.V1) { + loadNextSegmentV1(); + } else { + tryLoadSegment(); + } + } + + private void loadNextSegmentV1() throws IOException { + // just read raw data as input + channel.read(dataBuffer); + dataBuffer.flip(); + } + + private void loadNextSegmentV2() throws IOException { headerBuffer.clear(); if (channel.read(headerBuffer) != Integer.BYTES + 1) { throw new IOException("Unexpected end of file"); @@ -101,4 +140,21 @@ private void loadNextSegment() throws IOException { } dataBuffer.flip(); } + + private void tryLoadSegment() throws IOException { + long originPosition = channel.position(); + try { + loadNextSegmentV2(); + version = FileVersion.V2; + } catch (Throwable e) { + // failed to load in V2 way, try in V1 way + logger.warn("Failed to load WAL segment in V2 way, try in V1 way", e); + channel.position(originPosition); + } + + if (version == FileVersion.UNKNOWN) { + loadNextSegmentV1(); + version = FileVersion.V1; + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java index 9ca700a62f7c..5f7a8cf6ff8b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java @@ -123,8 +123,7 @@ public long getFirstSearchIndex() { } public static WALMetaData readFromWALFile(File logFile, FileChannel channel) throws IOException { - if (channel.size() < WALWriter.MAGIC_STRING_BYTES - || !readTailMagic(channel).equals(WALWriter.MAGIC_STRING)) { + if (channel.size() < WALWriter.MAGIC_STRING_BYTES || !isValidMagicString(channel)) { throw new IOException(String.format("Broken wal file %s", logFile)); } // load metadata size @@ -153,10 +152,12 @@ public static WALMetaData readFromWALFile(File logFile, FileChannel channel) thr return metaData; } - private static String readTailMagic(FileChannel channel) throws IOException { + private static boolean isValidMagicString(FileChannel channel) throws IOException { ByteBuffer magicStringBytes = ByteBuffer.allocate(WALWriter.MAGIC_STRING_BYTES); channel.read(magicStringBytes, channel.size() - WALWriter.MAGIC_STRING_BYTES); magicStringBytes.flip(); - return new String(magicStringBytes.array()); + String magicString = new String(magicStringBytes.array()); + return magicString.equals(WALWriter.MAGIC_STRING) + || magicString.startsWith(WALWriter.MAGIC_STRING_V1); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java index 20ae99754505..d922aae2e991 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java @@ -31,7 +31,8 @@ /** WALWriter writes the binary {@link WALEntry} into .wal file. */ public class WALWriter extends LogWriter { - public static final String MAGIC_STRING = "WAL"; + public static final String MAGIC_STRING_V1 = "WAL"; + public static final String MAGIC_STRING = "V2-WAL"; public static final int MAGIC_STRING_BYTES = MAGIC_STRING.getBytes().length; private WALFileStatus walFileStatus = WALFileStatus.CONTAINS_NONE_SEARCH_INDEX; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriter.java index 7015d45c58b0..bec9979be94e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriter.java @@ -28,8 +28,8 @@ import java.nio.channels.FileChannel; import java.nio.file.StandardOpenOption; -import static org.apache.iotdb.db.storageengine.dataregion.wal.io.WALWriter.MAGIC_STRING; import static org.apache.iotdb.db.storageengine.dataregion.wal.io.WALWriter.MAGIC_STRING_BYTES; +import static org.apache.iotdb.db.storageengine.dataregion.wal.io.WALWriter.MAGIC_STRING_V1; /** Check whether the wal file is broken and recover it. */ public class WALRecoverWriter { @@ -45,7 +45,7 @@ public void recover(WALMetaData metaData) throws IOException { if (logFile.length() < MAGIC_STRING_BYTES) { // file without magic string truncateSize = 0; } else { - if (readTailMagic().equals(MAGIC_STRING)) { // complete file + if (readTailMagic().equals(MAGIC_STRING_V1)) { // complete file return; } else { // file with broken magic string truncateSize = metaData.getBuffersSize().stream().mapToInt(Integer::intValue).sum(); From e70df23c5acc41d24b7d5d93d4ad822968380010 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Sat, 11 May 2024 16:23:01 +0800 Subject: [PATCH 08/32] edit configuration item --- .../java/org/apache/iotdb/db/conf/IoTDBConfig.java | 12 ++++++++---- .../org/apache/iotdb/db/conf/IoTDBDescriptor.java | 7 ++++--- .../storageengine/dataregion/wal/io/LogWriter.java | 11 ++++++----- .../assembly/resources/conf/iotdb-system.properties | 4 ++++ 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index d3d1a37e8977..63e0b59a04eb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -49,6 +49,7 @@ import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.common.constant.TsFileConstant; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.fileSystem.FSType; import org.apache.tsfile.utils.FSUtils; @@ -1138,6 +1139,8 @@ public class IoTDBConfig { */ private String RateLimiterType = "FixedIntervalRateLimiter"; + private CompressionType WALCompressionAlgorithm = CompressionType.UNCOMPRESSED; + IoTDBConfig() {} public int getMaxLogEntriesNumPerBatch() { @@ -3972,6 +3975,7 @@ public void setInnerCompactionTaskSelectionDiskRedundancy( this.innerCompactionTaskSelectionDiskRedundancy = innerCompactionTaskSelectionDiskRedundancy; } +<<<<<<< HEAD public TDataNodeLocation generateLocalDataNodeLocation() { TDataNodeLocation result = new TDataNodeLocation(); result.setDataNodeId(getDataNodeId()); @@ -3986,11 +3990,11 @@ public TDataNodeLocation generateLocalDataNodeLocation() { return result; } - public boolean isEnableWALCompression() { - return enableWALCompression; + public CompressionType getWALCompressionAlgorithm() { + return WALCompressionAlgorithm; } - public void setEnableWALCompression(boolean enableWALCompression) { - this.enableWALCompression = enableWALCompression; + public void setWALCompressionAlgorithm(CompressionType WALCompressionAlgorithm) { + this.WALCompressionAlgorithm = WALCompressionAlgorithm; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 9dc53ba6a2e7..a3c4fec1e19d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -62,6 +62,7 @@ import org.apache.tsfile.common.conf.TSFileDescriptor; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.fileSystem.FSType; import org.apache.tsfile.utils.FilePathUtils; @@ -417,10 +418,10 @@ public void loadProperties(Properties properties) throws BadNodeUrlException, IO "io_task_queue_size_for_flushing", Integer.toString(conf.getIoTaskQueueSizeForFlushing())))); - conf.setEnableWALCompression( - Boolean.parseBoolean( + conf.setWALCompressionAlgorithm( + CompressionType.valueOf( properties.getProperty( - "enable_wal_compression", Boolean.toString(conf.isEnableWALCompression())))); + "wal_compression_algorithm", conf.getWALCompressionAlgorithm().toString()))); conf.setCompactionScheduleIntervalInMs( Long.parseLong( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index 44ef2f4e1965..ab7e11e911f4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -49,8 +49,9 @@ public abstract class LogWriter implements ILogWriter { protected long size; protected boolean isEndFile = false; private final ByteBuffer headerBuffer = ByteBuffer.allocate(Integer.BYTES * 2 + 1); - private static final CompressionType compressionType = CompressionType.GZIP; - private final ICompressor compressor = ICompressor.getCompressor(CompressionType.GZIP); + private static final CompressionType compressionAlg = + IoTDBDescriptor.getInstance().getConfig().getWALCompressionAlgorithm(); + private final ICompressor compressor = ICompressor.getCompressor(compressionAlg); private final ByteBuffer compressedByteBuffer; private static final long MIN_COMPRESS_SIZE = 1024 * 512; @@ -58,7 +59,7 @@ protected LogWriter(File logFile) throws FileNotFoundException { this.logFile = logFile; this.logStream = new FileOutputStream(logFile, true); this.logChannel = this.logStream.getChannel(); - if (IoTDBDescriptor.getInstance().getConfig().isEnableWALCompression()) { + if (compressionAlg != CompressionType.UNCOMPRESSED) { compressedByteBuffer = ByteBuffer.allocate( compressor.getMaxBytesForCompression( @@ -75,7 +76,7 @@ public void write(ByteBuffer buffer) throws IOException { boolean compressed = false; int uncompressedSize = bufferSize; if (!isEndFile - && IoTDBDescriptor.getInstance().getConfig().isEnableWALCompression() + && compressionAlg != CompressionType.UNCOMPRESSED && bufferSize > MIN_COMPRESS_SIZE /* Do not compress buffer that is less than 512KB */) { compressedByteBuffer.clear(); compressor.compress(buffer, compressedByteBuffer); @@ -91,7 +92,7 @@ public void write(ByteBuffer buffer) throws IOException { */ headerBuffer.clear(); headerBuffer.put( - compressed ? compressionType.serialize() : CompressionType.UNCOMPRESSED.serialize()); + compressed ? compressionAlg.serialize() : CompressionType.UNCOMPRESSED.serialize()); headerBuffer.putInt(bufferSize); if (compressed) { headerBuffer.putInt(uncompressedSize); diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties index 38f3f7c00f03..7c64edaa900b 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties @@ -1431,6 +1431,10 @@ data_replication_factor=1 # Datatype: long # iot_consensus_cache_window_time_in_ms=-1 +# WAL compression algorithm +# options: UNCOMPRESSED, SNAPPY, LZ4, GZIP, ZSTD +# wal_compress_algorithm=UNCOMPRESSED + #################### ### IoTConsensus Configuration #################### From 2db27b14f670eb146daec654546c49fa5251c466 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Mon, 13 May 2024 15:23:42 +0800 Subject: [PATCH 09/32] add log for WAL size --- .../db/storageengine/dataregion/wal/io/LogWriter.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index ab7e11e911f4..dd83d48f5d7f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -35,6 +35,7 @@ import java.nio.ByteBuffer; import java.nio.channels.ClosedChannelException; import java.nio.channels.FileChannel; +import java.util.concurrent.atomic.AtomicLong; /** * LogWriter writes the binary logs into a file, including writing {@link WALEntry} into .wal file @@ -54,6 +55,8 @@ public abstract class LogWriter implements ILogWriter { private final ICompressor compressor = ICompressor.getCompressor(compressionAlg); private final ByteBuffer compressedByteBuffer; private static final long MIN_COMPRESS_SIZE = 1024 * 512; + private static AtomicLong WALTotalSize = new AtomicLong(0); + private static AtomicLong lastReportTime = new AtomicLong(0); protected LogWriter(File logFile) throws FileNotFoundException { this.logFile = logFile; @@ -61,7 +64,7 @@ protected LogWriter(File logFile) throws FileNotFoundException { this.logChannel = this.logStream.getChannel(); if (compressionAlg != CompressionType.UNCOMPRESSED) { compressedByteBuffer = - ByteBuffer.allocate( + ByteBuffer.allocateDirect( compressor.getMaxBytesForCompression( IoTDBDescriptor.getInstance().getConfig().getWalBufferSize())); } else { @@ -102,11 +105,17 @@ public void write(ByteBuffer buffer) throws IOException { size += 5; try { headerBuffer.flip(); + long before = logChannel.position(); logChannel.write(headerBuffer); logChannel.write(buffer); + WALTotalSize.addAndGet(logChannel.position() - before); } catch (ClosedChannelException e) { logger.warn("Cannot write to {}", logFile, e); } + if (System.currentTimeMillis() - lastReportTime.get() > 10_000L) { + lastReportTime.set(System.currentTimeMillis()); + logger.info("WAL total size is {} GB", WALTotalSize.doubleValue() / 1024 / 1024 / 1024); + } } @Override From b505d31e4c3abf3c10fe3139af123d51ac1ea38d Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Sat, 18 May 2024 17:00:29 +0800 Subject: [PATCH 10/32] temp for debug --- data.zip | Bin 0 -> 37702 bytes .../java/org/apache/iotdb/SessionExample.java | 64 ++++++++--------- .../confignode/conf/ConfigNodeConfig.java | 4 +- .../org/apache/iotdb/db/conf/IoTDBConfig.java | 6 +- .../AbstractNodeAllocationStrategy.java | 7 ++ .../wal/buffer/AbstractWALBuffer.java | 3 +- .../dataregion/wal/buffer/WALBuffer.java | 5 +- .../dataregion/wal/checkpoint/Checkpoint.java | 4 ++ .../wal/checkpoint/CheckpointManager.java | 3 +- .../dataregion/wal/io/CheckpointReader.java | 2 +- .../dataregion/wal/io/CheckpointWriter.java | 4 +- .../dataregion/wal/io/LogWriter.java | 10 +-- .../dataregion/wal/io/WALInputStream.java | 66 ++++++++++++------ .../dataregion/wal/io/WALMetaData.java | 27 +++++-- .../dataregion/wal/io/WALWriter.java | 24 +++++-- .../dataregion/wal/node/WALNode.java | 7 +- .../wal/recover/WALRecoverWriter.java | 4 +- 17 files changed, 153 insertions(+), 87 deletions(-) create mode 100644 data.zip diff --git a/data.zip b/data.zip new file mode 100644 index 0000000000000000000000000000000000000000..876ef4eff2ab0bacea5c5683453b9858490044c6 GIT binary patch literal 37702 zcmc$H1z48N@;43ADIJ1zcZhVCv~+iOcb9^6m$WF2gh&fWw@7z4NW=G_uN}wte>m6o z=n7ny%>M3~+1c5dxo4%tLBXJbu0KKnrAi;a`R5-*ARHh)Z98pRT}ulCV?zr|J$+gQ z`CCAsEv6y{Yk*A9emW=ybJ! zK!rYP{#T5sr;fq?0E`QOh4Juzf|2e&!eVRn0xE=s4ZBc(0M5m4)EINs43k z8sBP>4jG_jFFOIt3EPEuONyV!wmABZlGr|)bO?Cu5PCW)r6M^sd^#$&gZh=02G#)A zCwuUfD*=x*?|T1FxkCRJTmb?OjiM73@ zo%UZUuh+L$`+9wmpq^8c0We*U8`NE0GkaS*eVe~R1(^A+K9epTD?KAUHJuIvJvEay zn*p^pvkn_IvoJ7uPGP5@{ zw)iVxUl{h)bggNm7v%zkYY74bbdwCzwy`s|Gq(I&Tv}RsURsYc&(D$T=Ir<(nBY+$ z1>hh-QRK;WJxAsG-S8q1Ltwc z024QU4I);e6?X~}tf8Jqmr6IYdWR06-S+`Q>&AKW35|`V zr5z2xvobI?)BlqRJ-fQPO1su@2ndMkg`=}0K51q@=)5i|?_$ooY6(}2SC9odkohBn z13EK?GmwAKHvRwMlW1NG{BIuTdW-sMMi8~=o`?dNYz`PVXhuHot+u*G`sUjI$p<11 zD}nQ(g9Yoo>vb!P_fd;{^b@btfI8nt_3S6TF2Do!8{G^<4y(%Rb;=4Wskdb!F=a}h z_F*fGh=|Yv@o_X)U>wOx7Ytq z8ltPG=LPbIy;<(&z}o8>{9h$ItloRz0bm_F;E;8LHuO5k@t2T3ZpRStP_4}kEpP_b zSXc?`m_QkK02?x>?-Fx5Dx*U_B*7!ZC?^aIb}gzr-VF$?3cz34iK<7=;L$pbG>711 ztsx3xp`c`~0qUdN#f}79v$G(#=;`w4cz{u&4G<3ZKkOqMfihI1bW6xZMRnQ9$#$mW zHQj;v#_s(Xw7G!@fv&xcjlPB5Uxr7n0mOPQQQ-iDaXoI}B=Z@7jkbXuwYk2XHqEs{ z|0(nY1x$zmeH$8>4jF--ft7|1@Q0p*o{p7)jt%6ip;7=qWk%ZveFHcojAPx%IpH%@ zGfP8&+5mPqLqI&kR+H`{*z2&;FG5S3ajr(H3OH@O)vV!3QD&0ZgKg#mMN{suH!f!jt%Qp zXOqLFFT1BV9I9)&RQVmmWsEL{>jp30oW4(+a(aBVw=dIeVAL8M{)EF1%r66*;Z9GU z8YW>Gc*`Am4hV5h*Hvh~Fkmd3Iny*C&|?OvhHCdlBo(9dna8lNGgV>y6%Qgf-}A8_ zgtKta3PUoi!w z@WKkQep_UzZa-Un{4J${;!5E`gSrnJmGT(wp z=WvfH$cno&2)H%D?1@fWci5yj5cPHU$=GOKu#ko6K#qC!Gho>>oglWSFieFeJQ@u(s>e6>#i_D9VzZ?lScBzEyS>?E6^iaf#q?SS=}XQIghUHnTrQi2|n}k_IAoyp2S53MWSo3*==7a zs+WT;Z8c7HLqYK6hlRa7>1!r4%9Ngovp&8;qqAzaDDbS0o6`ctDRuQvz;`fCFAeqF zSvWJoDVJe`rMC)l-<9Q=B%iQHb{c>9mlKj=7#CrU3Zq}*D?bCg{VX{c#>9=G@qZC!ElBb#67t;`1) zm`s`!TPSQAG^-Fdp|=S*1=x?of(ssJo9V7rqf@ozKO)xBwa%MlC=@sz@FR4KgfVQV zc2!~J5M7M*o)`$+p78J%I^Oeqw6oazf~!E_7`iI=RRjL9K=97vc&J`N>LuY2&st*&~)0S~A+C8KX@kc3+aO^oyH1}>IK5ggVY#6|L zymOG4Ze=6a9n9cO|6*;Z`~LE3l?ldjjl;2*KwC9B9ekh$&Lva7^^3vrHG>s4gfLtD zj(h$4=wHJx*E@ic?jarn!1`Yg>HpqA_-t0c>;PYk|C^t~q9P|tX{BrUH40H@k&<>V%`g%f^HuveE0Wm1^`}EM?W1nW>0=omieLckfd+dNX z1K`j@V`iyq`UUm;kxnb_EP%#7`w?467)^z<4f1qlkud8qDpl|b){59}jMKI!=e`F1dr)Up&@sBfqRM?!mHK9Fcne>1(CITAe3yS3Rj~ zCJDo>RH{@OE2FQv9xA8wV!Ep>jB(`dhr}vK#tpx!a%DHgeI&GCLLEWViv7sam@{=s zV2~}N%w95zteWIGPb0P6fYB7jCQohSZVIyz*5G~)dQbKn2Z;@VGH?>ON^FM$*sDUi z!wsm29DE#~MHMm;RVOt5@G?ydV;ZKgR;I@!qNrB+?%ucWqnn#Hn!f4q6TEzUOAOqN zlu8Vi1|i0^KX-oRlm!}{J$}x$xCcG!!6Lu%{Wt-REui-VClvQyNWzf62ZjoH3?o3v z3JN0Zt1;4nA0fpV2TIwPvRXYwFzUOf(Oin}gN$QG%4~(k7MC98PGrdNDh_^dWsto2 zWj+E{J>(QIH~ihlG*x;@se0VSGK%STW@xFNEbB$ZJGhC!k1kl#WM3zxaJ(z=HTNGc zrA62uuDfdYe2Ol5nY1~>rxPEBG0jtk>1mmAL)z#HT^&2 zuo*t5vA=@&#X;liL;lFNFR|D!?ElKP?;rB7Kzz<;|3_y3h~rBv;l^;#Ytl1(V#*)D zeA-a1CDW=bZ>R!LC?=?H^2s*~#tv{cq_K0dqxmj}bgehP5SgwPAgllw~F?KXUNazO9N|Rw#&ky)G$TF=*pU3nac$`GMTS%Pf!#q zlJ8yy0uxR7Y0zb2>)XDD?C@Q8?^ZsV?sM)cZ5qsl$ajf{n&&UFjuDn?8Q0wA%c@S+ z87GEOUj4wfZ-aKu{~NY_4B&mY27epC`wct42Jk-F-XAT{4KKRSzWPVU>Njfr)e>EU zG4@d$`W!C0VdeV_=F7q1Yv|}2+A?7L;9!0Z9nt>~I%57QbR-t7x=;sLFX{m2Q}T*3oOXop~W64N+nWQrEiOry|*7T79)k{%q!BeNAbG~qLp*%CRBIdnw; zjA#I`c96==@&-^Sb5f$zQZ#R=$U3@uecNYjVT2nuDIo}6S^-Tqz0MYCb^#%ZXd@r) zXjcepWzHeE@-~Oi`LJ+4;S)W?fQXI}MR2c*m#;b)jX0q?pI{sgZpJ6US(;afC?X*F zd`^d;TP<1Ehr6oE&Kz~gE4dxqj?ge5Ag~iZ0;M|pHG|EhuAVgHs=m)D?yO=2ZWnALM6g^lu2aX&lq9GCZbo}OP zJFGo53lsn*;WgIR;00#S=Ic9dJhdN{J~IOh0r^r-RFCPK0PgjI7#>pr`Fwi*lBT$s zUFfrBeOVA+mTA||YvGqCY4%^7r0IV?NnbyQJ)%$S^(p##DBkP}`SDbJy_$c0s%EGA zyHoX#X7TPH0AI`EzjOP@{*pRi>-_4|{lZ*bThg!Y3-%p;2?7AKDFNf&znFnu^V1R8 z-bVX6heZQ;zAt+5i%I7{IVB5PS|Bp81U}djwx6>dN>ZV|-zNQkbnuXW0_CV}_8vsbyP3dg_1pcCFTPKdbile49DXW$6UW;tT@>^yL)xJF=Pp+H`VE`>|FGdn+%uigzU?HGtY)mhEBgsC>y}IAnbxn0bBAEHdb@kKJ>$^@|J@e4;%khKE)kRj_RD$_9` z4$2WgIYEGd-r*rbgzy7J;<@Vup6O)-1la=A>#alp9|Vc)B|vaX1VYwJo&Zh+QUeH+ zhhzu29i-G7hXB1DR0H^g2Y3g|33SH$CC?pcFjP=t9&%}jXizjBVrlSKpatGo>9_Mh z*SwE_fQ^8#yh+kAjeu3WS<-P=fVhBCc+mUc4uI-_kXArXy~7(wwSX|d0=%?m80kR5 zKy(|}=z!gTNxaFmu;PIYyxFmgj!^G9;f3Hln6k@jA*F?zFrK4E=Nd@S#|$%{8x;w< z(ob>UMK+BVkCEon$xGVYe{vBBLO)36v2AfoQ$CC_KREUnFC<8b^9`DW(+arVRI!8! zLI>94AuEa1<|*==xMfGaJ^{LCS3ATMk)SGFt+cyN>J_WKBh7vj^;(z3bTi=$87ZE= z^r-tSZiv{6cjrq{n4E12s^#V{LGA#8`Qi-Sj|gnB7ab?r*_F%L6Q`0MM~5RXEX^(5 zcbthK&05M-=6DvvOm*%cRgb6Ks=rM{u_2-rWmSM2tdUGtW&g|*vX)dVfPt+y#&J?y z`5q~ve^+ZlmW|yr{cy#C7^H2ZCBt-LQ+Tu>_+3iH6c|Uw567CD?v7nbGI=jIOynXm zo`?xIKZ2+c>3x`#hZh=CCX{JdO@;y?!av>i;x^T8vbN^ zt5?aLjte5U9;p-Lm{JmUO;+1+by=8=O{SP%SrBcb$9C2%yRv5%lmu}m$j;Cz^eY7E zOMB?QA&6kkI-uKYC#(K{5YYSXLCS+Rf)ZG8feilEy%>3KYI3&h3zB(5knUR}QVNfx~%$yAf;jp>z9908rWswNC!V z4y(7Wg7}WFa^s`d9&7QI>bPq<(~#t^=!KK-+cy^KGBQ4k5uy<1pvJZ9+@qG0i6b8t z7xbUA+siJwU2+@VtyExC3+I`!+={Dw;xP7G$S3)<<#Kr4Jz7OipYRSX`OmVPbPC?B zGieFZZdWr2-~~(4FY-U}7;WC&bIhf4Z#pSAZ^i2gE2*b2Pp-|;-EYLbW6#*Khq?$B zFUfb>jOAPqMoNbU!#LkaD?WR$ww-P{YQfyS2w$i2R)*FspUdQ~6&;Hk!5m(b2<^+z z103aU?X|%U)*>|I+htP<6v2KaN@yBYeEmb5M3>`|$QkN-nIgMzB!WC^^x6w$yL9UN9NKDREAHx_-MO_VN83m6hlh=-% zaf*_Ib1Oa5WEILZD`$vIlWLS6`0>clGPBt4)ZH$bO{X)Q+||Q$nUMk3M1EwF(y$G| zH6Yn2+0SHRYSQ`Ie7*S91G(LGiqN16yGS zC$vimYW#JtGuK%U!WhrJyq`U{r|GMbMz7C==k71ab){@%WkEZRiay#X%RBACXI5~u z((R!FlDk41Os5#U^qrrJ=;~vYm@pl3zb`W43_fa5H(?vQtdV_MRY|~}kJSI*6g8&+ zF8Bob8Ki;{Tn%s{)qb-cB|=Dx+oldeqEFK8__i~O#`fm2bD!F})bRuvs(ERUTx=W| z6f6rDg4vZ~*%J$su-?0Oc6n&@Rc#fOOH?J}su*SWhDcJ+snVq$njX8O9iCrKt-lwO zt+>NdHyfBHjEqtzkPO3YY(V!I?yzNXg}=**jZ7g?EAQspvI?NE&SfN@PTjG*S~s-VY%)&l)O$)fr8Rt(ED4l~uPzUS7Ui(Qmfca>>zr`joKt^d;B*G)C1-RcIW% zk~I2$`@8Ya)Xex@3k08gm7qA2Yl%!y8%R7>3KW0gnY7&Fq^+Ivp!m!;7SVOZCoze) z!39EdhO`8&F6qKY40K)(t|Gp`)yeOxr|;5*nf1 zgZyCQ*i>Hl?dUuo*W-3>qvU-eZdbAW3x6Y#TJ&9Ze~%;Ig86bHgA0yFA7qoLFDXIk zZNX*krxh0(kIq>nn@g`!jl;^Vxlu(*oG^<-Ar{uK(@dY{csL4x_@1o#!tZUbR?xts z(CsP~7cdn*cgVkOl{X@8Ji?_etXxf2o~=;Z_9|wQhpCc?NF4N~h&c9~eXnH65!$2V z`qD-Lh65>d$@w%w*;!Eh)7(DwSY*yNv~n3sr4o{R7OZu8%MY1`gPeQ#eRf%OL*Qy( zFdzq3t@v~QSB%5INW^bv5TOvg!0 z*de^?Ts?=QNja&b+#}H8=>-_jHS7EBp}znDaY{|-PI?jKQ?@*gWFc$oY*;fPiw@s9vrREh_;aE zwwZhU!T~2SlQ_{aO>LoBB{eF<1@{$KFDfD!{>9Zf?UfWw)Ro5874Y}ovcxK+aR@9B z5HJc55a}-+`fr?YO*(42AAIGppWX4zcg{>huJ{eR(_#8T4stXQL~ps8Fdh-h>xxDO z@s319r|?yZxX=vn#kN%Vy-BSajTMcBx-&+Too6Jbmk)J+y5{BUBd%We9^0Oa^^^Go zTaD#c?n_~5CtagBZ^U9m#*KY=5r-ha8oMmD1*M@L2ZAfZ##iC)Lj{q=5f=22a*tAUsWMZ2xVLL;=b(Ic7<*_ z2hKs1(2ggh+ocoK4rxRU6!jtpwq=wMjhCf+>DI|z=(pZWId-Z~ta^DVmNhG;H{-9b z{kAbWFds+`i3BMJxyzg`rk<3~z}GPH0a_{)GAELWl~{n>;gyF}rKfVp>9i&^IE^6a zOIdPpMp@!Z zSrUlu;I#@hcq@motC>z+tHwk8K28!5AK#q`Y($@NZADBw_Bo###3Q~PNGcx`5Ipu1k(UUvz=%^E%2%Nj&M8N8?nAlpHqy$E^0qP@($7}7!VK<@Iu5rD3} zE2`IIgH8uF0#Wf|DF*EWwgbMeMOqB=4o(`_y#a_0EUZ`)fw%|b9ZWpX8IYV7w-y8r zP!}-F41v}yC19ootYToZ5=~9fWLzx-HlW8|C^N7)5NbdV8{pVLsz8N-5oR!~aCX36 z0a-Rc?tq^Ic@h9Yfk^?8dJ_;}Ks6HDR>_HcNZmTbo2GhIHCa1GzA3-GGH{VB{Ax=* z!_-h}ScP*I)DWQ)n7}s8v)0`V@&3z-qeWn%VUOBfzwp;*8jKam*5zGz2w2Tpu`2nE z1P-KhxYQ%A`|ry;WF5RGw7pxKh*J6M@X1C|bsosn;@f9Ygy=i=lZHwR>DJuF90M++ ztagiLmym)v?H4T4Grm2q-!aK&*td*#ujBshAiGiVG7bFsl%992Li%D{y7!%PJfS?9 z)$`nfXBDvqra~~CNsUj#TPRb7#}zpf^L^}K*F=4fiuMkLAjmq9?$6DKAbT@1W^jwC z({ivH@kDp`Z55+>ypwjV-t>4nMiU5U{TBZCXql2^(dRriVIRE2RRH4&BmX`s5qGuP z81;g`GPeBjsoNXKuJYYs!#tbaVqEt4HW2ze^;r3g(j9?;J((+y{)}WPa3c&pRFh$|Tel|lU8e(u5d1@Ah0Q)_mTV~2Y1IBF9vW?U zIQE*L;h5N`60Xe^fk+XKgC-qK976Te7k41#9+_x3?`8xLb19W(3Ugb#&rdL zTeM0p>PL@VQA|zMUy71Ed#wx(rZU$M*w~O84=LJ}EJS};l6OQ+%zS!wP*$R4$%kH*d3>1&Zl~4jHaFnH_Ct}+SEyfz0)tUtJ73iUNI|p zp@4=y4TVpuVm9wCpVC?r%ztRx(=xdFj*R=r!7ftr^i-^ivO*W_*zgtx-xkk6*Xv%8 z9Cze;5vcZeJ@^Rd3ANL%B#G42O6)b%=PdQNB!|;SI6`!WH*kt9ukyKez#nGP7))n? zLDx3N+$y(A_Nxlb87#M&Z-g*DEK8ak@V_uj9*mLWTjo=S+XxwDbhCwyPkC8{I7o!& z7U7mSX-|>Lhdji8*3FKU744$L zSda=A8xgi-B}lw5n-PpMIF6DB{unIuG59m2Y5FJd{3lUk{ZY}pG>%y>iVe0!xL!a5) zH3f$uN|918{vaino)#HSC6~9E>6za7tUo2`IqP^D`Sy{@ETXKHBUmzm5*P$eI^t8Bvn0!kYq(iYOax6~_K!bx$7?$L*3P z#iJFdo^3jbB(HqX<2W&(UweAjA_aT5Zemd!FGn6P0{#GLSe0rhE*NfZkX8zwc?XTL z%PQ$z>Z7VxuWAX)H&rxR-t@qhX35mf2)gkK&c7&OwHB;Mz#aq>AF*ez+A7bm@droE=%(Cvr>>dBoeby~8ko-RJIh29mZYfV$ zpHbt&z1pzKHb-THg-m9Tp2vy~lOKxTE?g2bb310rO}c4fc9#fHY9Nh|v}5_5#TwC!3D?jf3#Twlp3hqP4;J|9k@~KD{Gu~Iv-q(UU)ie44yk(JhG%hyK`nwwTru|MiQO}y*an%(_fKDh zu&N!}GU%V%W$w&Pq$@17Ba`^el5el|-=m*=Yp)}>>dgP-6;`%sSphQJ9%4n4kMixq z8Ri$h8iJPS74r%qQ$%={Zyp@1;*X7UICxfWJnm{RR?S>kM5j{q6kSapJjRU5nQXPp zeHyR>ApH19SNp-S;)3yR9$lbXmfwZk_uAq|~9C}GlUtP)1 zs)Nu1{hUkQk*tBJ zWu@|@0s$F%Id(45*#lemX^N?|i$(iv}kz;FJ(FVy2wTYoQC$z$TS#w-Zp@^ZLi{*^d&UGfvj)Z1t96 z788Ba(7E@FSjKA^?DDd}EtccbT4cilojq}=>(lqg@^u5n4=vz{wB))QJrkj|EG;`p=dAOXQJt@PZEm%OFX4`m-CJOB3*J@8 z{lMmUuDw8kiUD)5I?W-((QJh$p{MARLm!TjXw^kDQ#FIQ&#q3TB_~B*CQdl@wDbaAK#)Os%JClR{(U3>;Q(H@ydWiHK$z+;|K5y;UpI7se zGe!opSq=b(!}(OHm*f+TD=O5U&zLJ8M{J zX9_mYax1DMj22#ELtBBKUv7KeddR(ZIG@Ws*J6w_M`$$jKfl~+CRve9+5?V)%q?3lMZlT;_$zSnKI#Mdp(GJE+4a{KOC zysB3jHHYB#$3;DGQH{6gthcDFMu8Y(c*WZePgo=^m+N1f8gKro+Xaet&~qpt&aD^}Ca z*j(TCx<2fqSg+^k_!&3-?|~EjPOkq<@@A8)NTKmVp zMY*JZ9cGy}PZCw*b3NtvyPmx^R@dSq0Z^Enqpg`8}y zYqduU#=4Ap<`PEtOD$8$sjW!+bg2#AW_vjnzq6*D=QIP|#R%ib2cTbt^y3=&Qegk1 zc6}=J`aiCVF9r6$q55AgyWg6@e?WgBPxlr7FE+nxu6(Un__pAZ_IIlL^8}xZD8E!G zT-(1N)cAi}JNc<5;YYK9`MXB|yxRJSD}QA*zL+7vH$Rs-{Ilk+N!poPeI@&wrThu$ zkDD6GPe`w|<`c31xI_N<*m%DtW)EoT1$f8ULie9nM*d)LK7stt>*OQI@2UQ#{Nnek z@8@;$J=HIY{HN-U-;@7&o&1LUHx(*>-2Z+Q-Om*&zn$QprT7Pdep-sxQ@jnWvLXj4 zoHSra{92%282^Pu^t5!e^sMwObPRwG9RnQ`HK0ikH64we?R9Sl8i03XX>X(Zsma4{ zd=x-^s+Jazwq68m%Y@*`39(D-YJdxQ06_^vp4x>c7@+>yXDuKh;@fx7ui-uj3L<*} zfZ7Gvm-uf6?iWw$7qxIOdHNjI96BAbiyy zPsOWDAi$6HfFblRXZfl^j0}K7z&9%NCz-@3*-N8NjB42M#zxZc@?wF-T0B8Uiiq8c zgfLdJd9oE55g|psnyp+T&f%ymo5g#4{`$y%>8NcxZM$oN-LYrFdF#I8($sRY@Sx<> z$P_Xt&(_+)LeHi_cc*z{sQ>DlP>_tL8Jb=Wyyyh7ctAjiQ0aX0yr4jRbA*oV6y$i*It~>);$rGbbkLG*WnI6D;;1RV9;`mZ3>=xoc$cnX=A$4Ot6EH8I?Im@X zCf}39E)>*cub&hSPKX55os6$FEQITu3Tml6?I+WHbQbL6VGS$<>gdLOWK$q`b`mT; zO9h`7zOmA!g+;2qC9`B>QP8>EFmynTk{KY)5FoqfA%p_JRansUtqf%~~lAv9(ug%Tk3kEfrWOs5%WfIJaPW9=(QI~H94eux*C z*_?~0^R*oK}PGA<}0G|O(tc?0< zN^f{I>qYG;J^8FRZX@B`_<4PbSwoaQqPJ8jgO_5V3)ta{J4f2SqebE_1$5kohjp6$ zl~ENB%y5iOAX;C^`{ld5mJw&Le5vIM4j0@96t|3T2hUTaZ2>-A-pg#qznJqp*GlHR zbgg1nj!=b3BUwRs%o1q>qR{^p6vw3$g5*A2yD-c9NidDaSq#zDL1Yl!D!zg4&Z^dl z^jkT&M`U6xgeDXye8v+v@&KP?`h4F#OjKz97vxd%E_rQLA zP^^-2QGc&wMUiBz>YeFRssU2eSr+C(>s?-W5*fMW%C0BeL-A^7JfvQ}u_7;1rH*#{ z`VNzV&e=)Pw((?q{dyX@g6Whyvv^n5W!)x+0s`F7gQXHcO{ zHyjDLRJKkzMoOkgkj)WHhr8(ZSy6f<>0bGtK~~Rmv@y{ku-Vb^l9Hl|<|d4uonCrO z9dt6!9YT&mE}PNw8;$%0uR)tbL6`@nJ$ zO@4HctAAQ4@|2fW!Vf=?br?5`{e5WAd*jyP<-#l|Tg@E7aBRV_t7PZY`bMZ|8N2Lg zt@Z?S3W3rmD&AS*Z9?6N3r}$C-(#eb-$~(Bvo@m*NRr$She9h8A-7#iV9>#gG+p5i(4LQ<9=suU$s(d z%sk!075=K(_hHN9_cEpC?lG&jO43o5Y;&7S=AMrPVIoatcLX)H*@51{tM+I2Ts5ST zpt!Sh25RQrbKYO_Pu#W;qDL+01~&^QS8~cbn`BKj+$dJh^}!0p zI%u@+1F9fBBLz&wcqx@%dOam3mxKD=kEhQuTk z;tnyC$Q~|W-hI~pv@_ba_|2=1dEYhRbyYg(hD`z_RF3Do)5Ef`^_fY5#JS|e?n)Xy z&^45&x~ph`7|Z6d1o9@0a#-G!9jj{@(GZ}gF3WY5WiiuuUImhV(_F%y*=ci@+RA&ivP;ujBV17LO8ooCWh{KcKz{^NMNA zw(&ERC|$pl4TlKVoGHcv47Aq8l=D=FGHa45EjYCGop7iWP6NLpB+L7CD4C)EhI)3d_BLm%`k<%<+)R@Wo%-9dvzS1Rg^XN@*4F`ABU%j% zV)~w|967epr^WmQY>C2RwBlnY;*zd*X;=aB5AIEL*pWGn(BQwSpH2)?jtV$fN6NOo zkl2S!=Jp)WUO)3WSV~fJ?7TP|SRynj21pLqkIa&=irXuWN+e|JS;kQ^mgNVy?s&EA z;%a>CXI%3S^E3{$>47Qfts_LwC$Z{wI++O`| zP8EyrrVL3Bd+`WCK+TTeato57H~8#($F41@@&UtE8!G7sgw=a`n$CtC{t}&%8ERf9 z7DsT4p@lv`1G5s|h9OpaPKlX`@Cl2)?WId*NGAT3BnNV=cs-)^2|O@Yh3u(7=Z>?1 z7(t4+`I=xMym3@hOJgrxM}WIXDgx+E{kuask8jDV!^-ws)rReKEo4Pe$y*NciNl-| zn$m(hoN06_uAW!>+AFCZJ8qo0W|BD0^k;ED$xGOA0;dOZYEC$=3be^LTF>P*xDsQ1 zb4RsUMAw2Yyfrl6WibQH1KpG+Dt54g0(Ty|giI>=626ISLkTPK$p7*43LGg1^m*2z z5wnr5lGD6`G9T2;#M8Gc8XLr>FK#`N-sH+TsR9pv!|8g6lT2Edt!Z9O$Fs^?A)@I; z9O(0#OE-T9l9HE3vCkWxO;l82gx7NABL( zU0=(+gEi$bwntKT-!^$I1hfkD_Nk1=R$1UWHHZ>hR{?KO$zBg)vZzbYtN1j93G z>a?zw@+2jfo&rDW&jd=g>Kga1*MQ#SW=PU%$7JT50H1~zMdsXm;oTpgHGwR|i8kP% zxQ7*;XT0B~bE2g;fw&24K!e%fLlW-((2G?Q)o@#R$`Yh2rGN(;XBOAipOgzfI*RbN zwMz6tG+2n1I>feh!Hcx0GVI+0s3+Ucs`E^Da5}^q=|%O>n3<1NAt@45Vm_>ToDMZt~t@-|j*R<==-8}HGKiyP9o;6tcJOd9s99BE7@^mm1<84hVs z;8ObL_M^p8toB+D-8Xx)2}0uDNl=l3_6TFnE|ht(VL>9IwsZ8G^q3o$s9%^{2gux` zx#S~T6|qp3EdEU^K`fDtiXTXx?my4^>JU+d+y1xEO zSfLs6XGXTexa=@@_?=Llj!wBAtZ0QKfUw?w{#=(BO=RC-Na`_$|9(*i_u3Ut>)8g* zZiLAblBLEY-INaHLIXL&=|vK=`vb$zfuUhgY?$0#sZa?kwullK^^r(5Kj^KmRcL#P z#T=IQR?e($9AS6&c9XKZVy05Z>c}$$`chcLL*hUZ4oUS#y-veI&8?1a*gdd$&w@7xaVSI*k# zY)^Fb*FvdGY{3`~J*%{Pf){d0xq2@)t)@v)zuWARPga>P%SGiGc`II=0(GZLBll*y zMmSk9`TAawxkZDDe063-9F=)UnI{?K`BfIq!^+wrd7S3jFcTlyc};O-VG6gTR6&2U zH8rEMYH)oM(O4xm8mXyDi^od><>oJYOQ>{oN|^a{aY+MZDSQ|0&TOdb>6Xifa>qyI zK1fsgSLV9hqIZG6auAgWUmWgx+BP~XmWWUNs3xL!7PK8D>lx-L)yM>s<2jXcxfmgl zf;o9y_-0zXVp(@B?qFN51jJxI;r!tqm(SEYTO1Ue_5wjo57f=Kdo?i(Z!i2>6HV`v zYIr|CKCOHa9E10A%|b0ATxhnZRRH|MqOSFMN3z3*IrctmgI){mdnLgepze2{g~_DT z7|r!?Ea?&;#m7DG=TQKEv>62(I}+Dq3qTcZ=CIZ%7*N4&f3$z}8LJ(V=hiy~0gDHy~fhh>kfCbE&G(4Y64 znF!Zjx^TRD$2i*8=T>)uymF6haWXnR+iklB=xOzWp*POnhbcdZ9Sy^{8o2@%*N3XQ zSxXR7wx!yoxU57gQj=IHPw!yO1-|i#u(Q6>T5t?RWa)0#&R2Q2nK~sldNk=-6()$1}O8@+KYQW$Gd9@Xd$fIw?LZ5cZ7#1lnacJczZ$y@%BC3QxY<{A)3IY z$wwsx9~nLb%e)_On@&`Dht_RyQnf}t8@wqfdyTCFGK%-sen3WVMEkTwlTewYrYc5{ zka;!QikQhR*yZcTL~vXyEePA=SAvO$a4%Y9(W4+qsCEmv~V1!B)OZm1dNW5 zF7zzgc7Qy6-h>Rnjqx(o-nG}^5fm~IyW!|l;#c+dB(3@h)Q(Eaar+WEFZve}<0e@1 zsuM~X9}cg)yd7zLa%v7^2<8+;Cj0?DR3i40=3xk@i@zr&esMWsr$$0ZtR;KDJfT{4 zJF*sOPlM+{^yQjuV2U(R|$p2$jMPlVFP$KgTWgXdR1 z@<2rlJx_=83cZh%?NFUG?hsxjhm87ea$^*>XMc6VtgRV~E&H=nOD)gFwAb^jkF@+h zEW3Ezo1JOpS_oEMvUIme?M+iNWFE5xft}ub+qEsj7ZljqaCG~g`rXDj?j0fYXFGAl zA21w}@5;_4z0@VeQ%&58BZE-2^<>g7h)OM(2)%`qip*lEyej~4JbNxN54B_=)H2;2DHvRSW!kXkKb*Ye*X#IBSWJ`kARi!=2R+3tmQ zbTj8nnEP;e-`4i%Xbqo^0?uH>0gb+JB(JCLDWRf<((X3Ig)qw5Qot1Tj>qd zDIwa6T#!L~1bx9^xkif#!jO}^6>bffTZCR62x(En}^Q8r&%{+C*p4L3p z!CU;hL1fUU7h+l5o(a-4fDGz&7=JlkW(Z-b~4qo9aB zM(&+p7G=v=a&G|Y?}y-cDdwRe-I{4F`#JBRSothNQ| zaOD>0;KSBF)LTCE4N54foAtv`Cs7h42G@VfHzj)uN-sf+5AU*sr-3^FzQJx@u+94l za`8M(^Ws_f`EbeQnyKemcB*Ugc|)*np{KRT)y&St!(`9U$okgZ>6}&CTOX2>xq08$ z$6Y}}+&f2rKdzrWzo@=^U0yG7%vy%Gfaq$gL-B!|Egid@+xdc#)cf3302j9W3KjE8 z=<4u;(G?9sx-ijc86^<*{x&gi))j0r58>(+9HAQ z2Y4|K0`S5&FbE##rx5LrJv$MBFaUAw>klH(ChzsvkMaFauju~M90&ZpQAqzn}`3;L8;(|DHbgcl1;LNdFIAWIlJYxz1etXAnQO&*u9M1Ueu!@FVNKKIC74 z_}Iz@fa90fvV0Zck2r+C!x42OIR3e&<==ym{SFK&pp@yqXUaz~fG%oZuWb1fu1`sV zYbJevVasPd2Lj6SPXC|a`U3HrF1~+<^!saDzC|hkX!vpS9Q@}Jm%jt|{p~5=f>QxB zv%Pt6|Ji7M9Cg1W62+(}{hM;}! z!2b2ZpuYq6?L{09zvEl=4TAgW29rO5`?_$y73%lkq#$pMb3fl?@+WX##riGhl)vNL z!p(vE*oo7{QgFhZt^YtfVcOBpP{;G~|q0QbLw4XaJ0>E7tF8+r|%l8fMKY;_B z`T?5;8{mika?|)M#4mjNTWNmZ%Klqa2md{)?^^KvNrb;vb@VUR;o6Z)!2Fp@-+E-f z)sJ5j$N7%<+V6>fd@JH_+4tlWt@;~r@eDC52vH-~XlD@w z5fl6XLGcsP*;@%>A%cG)q)9FK0a_@g2!frnXP=qbYoB?Y+;a$ANHc4{XU}G49mCO@ z&`~s~1xM*Y8A2R8n}j%gr+LY1LFxFc1Tk;Rcm?1NYT!Ebe(b0=eL6=7PmG_KtE$#f z^b_XAkq;cmc`=n(8Q>?q9+aOST5E=5s!g9ljnp^$FT(17Gplpk8Q3;W7)M`ng<-75 zngQ11v9?ao1hFy$VoA7O7D$^G&#>S7706Aj3l7mTg=UT1{rr5dMsghB`AFgFN8XdK zg0{T(D1G;b5zM_O68z*jw}z$!iPK#clB7Q`3}Qk}qi)1ZdOCYb#n^CbaHe*RT_Q z8(r5uyR+oh1|srKja#~~l_IsZwB3&q1QK1>!67%Ggj={Cvy{+CG6j~w$SuguA;HXx zKoy0G8~o`t;Oa1V(WlBdfj~kFiFb3Yga`B%IkyM@%Hm!MpDL9j!7v~lYcE#B>D`2q z0mNki-+AeXwgM{cbU>pA$Yt&HV{g1j^gD$YCM(f*>EXE6rpe&U;(h=2y@d6exr!#u zF#1&B$%F19*zaQj&+}*GfVlj&*)ybKN7fUHaS8yJhQTVF-gTVg0Bm+Yg=IeyUlj%# z0S9333kY*XI94e0QBGCco1($T4gUoQ^D;dj-eHrJ=gq4=9PGHL9?YE~^PpT7bxxh~ zpj4J7vxJ7sB9V){`106Wyv{Apj1pxC-4>ky4fa|PwkNFFF|=vU!NR8>Xw%q*foY;F zypCnemX~o1PkvyvNVZ<=0gZs!vRgpFZ0QJv*|NM5HJDe%Lp7U00Zh9+3|A>|ck02K z6?T+SPfHqGDe L<=fQ@R literal 0 HcmV?d00001 diff --git a/example/session/src/main/java/org/apache/iotdb/SessionExample.java b/example/session/src/main/java/org/apache/iotdb/SessionExample.java index 0daec0885bf7..beb81118ac28 100644 --- a/example/session/src/main/java/org/apache/iotdb/SessionExample.java +++ b/example/session/src/main/java/org/apache/iotdb/SessionExample.java @@ -111,17 +111,17 @@ public static void main(String[] args) // deleteTimeseries(); // setTimeout(); - sessionEnableRedirect = new Session(LOCAL_HOST, 6667, "root", "root"); - sessionEnableRedirect.setEnableQueryRedirection(true); - sessionEnableRedirect.open(false); - - // set session fetchSize - sessionEnableRedirect.setFetchSize(10000); - - fastLastDataQueryForOneDevice(); - insertRecord4Redirect(); - query4Redirect(); - sessionEnableRedirect.close(); +// sessionEnableRedirect = new Session(LOCAL_HOST, 6667, "root", "root"); +// sessionEnableRedirect.setEnableQueryRedirection(true); +// sessionEnableRedirect.open(false); +// +// // set session fetchSize +// sessionEnableRedirect.setFetchSize(10000); +// +// fastLastDataQueryForOneDevice(); +// insertRecord4Redirect(); +// query4Redirect(); +// sessionEnableRedirect.close(); session.close(); } @@ -403,7 +403,7 @@ private static void insertTablet() throws IoTDBConnectionException, StatementExe // Method 1 to add tablet data long timestamp = System.currentTimeMillis(); - for (long row = 0; row < 100; row++) { + for (long row = 0; row < 200; row++) { int rowIndex = tablet.rowSize++; tablet.addTimestamp(rowIndex, timestamp); for (int s = 0; s < 3; s++) { @@ -423,26 +423,26 @@ private static void insertTablet() throws IoTDBConnectionException, StatementExe } // Method 2 to add tablet data - long[] timestamps = tablet.timestamps; - Object[] values = tablet.values; - - for (long time = 0; time < 100; time++) { - int row = tablet.rowSize++; - timestamps[row] = time; - for (int i = 0; i < 3; i++) { - long[] sensor = (long[]) values[i]; - sensor[row] = i; - } - if (tablet.rowSize == tablet.getMaxRowNumber()) { - session.insertTablet(tablet, true); - tablet.reset(); - } - } - - if (tablet.rowSize != 0) { - session.insertTablet(tablet); - tablet.reset(); - } +// long[] timestamps = tablet.timestamps; +// Object[] values = tablet.values; +// +// for (long time = 0; time < 100; time++) { +// int row = tablet.rowSize++; +// timestamps[row] = time; +// for (int i = 0; i < 3; i++) { +// long[] sensor = (long[]) values[i]; +// sensor[row] = i; +// } +// if (tablet.rowSize == tablet.getMaxRowNumber()) { +// session.insertTablet(tablet, true); +// tablet.reset(); +// } +// } +// +// if (tablet.rowSize != 0) { +// session.insertTablet(tablet); +// tablet.reset(); +// } } private static void insertTabletWithNullValues() diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java index 603892e36811..6a7f91cf3547 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java @@ -94,14 +94,14 @@ public class ConfigNodeConfig { /** The policy of extension DataRegionGroup for each Database. */ private RegionGroupExtensionPolicy dataRegionGroupExtensionPolicy = - RegionGroupExtensionPolicy.AUTO; + RegionGroupExtensionPolicy.CUSTOM; /** * When set data_region_group_extension_policy=CUSTOM, this parameter is the default number of * DataRegionGroups for each Database. When set data_region_group_extension_policy=AUTO, this * parameter is the default minimal number of DataRegionGroups for each Database. */ - private int defaultDataRegionGroupNumPerDatabase = 2; + private int defaultDataRegionGroupNumPerDatabase = 1; /** The maximum number of DataRegions expected to be managed by each DataNode. */ private double dataRegionPerDataNode = 5.0; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 63e0b59a04eb..f8746ae0cecf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -215,7 +215,7 @@ public class IoTDBConfig { private volatile long walSyncModeFsyncDelayInMs = 3; /** Buffer size of each wal node. Unit: byte */ - private int walBufferSize = 32 * 1024 * 1024; + private int walBufferSize = 4 * 1024; /** max total direct buffer off heap memory size proportion */ private double maxDirectBufferOffHeapMemorySizeProportion = 0.8; @@ -224,7 +224,7 @@ public class IoTDBConfig { private int walBufferQueueCapacity = 500; /** Size threshold of each wal file. Unit: byte */ - private volatile long walFileSizeThresholdInByte = 30 * 1024 * 1024L; + private volatile long walFileSizeThresholdInByte = 4 * 1024L; /** Size threshold of each checkpoint file. Unit: byte */ private volatile long checkpointFileSizeThresholdInByte = 3 * 1024 * 1024L; @@ -1139,7 +1139,7 @@ public class IoTDBConfig { */ private String RateLimiterType = "FixedIntervalRateLimiter"; - private CompressionType WALCompressionAlgorithm = CompressionType.UNCOMPRESSED; + private CompressionType WALCompressionAlgorithm = CompressionType.LZ4; IoTDBConfig() {} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/allocation/AbstractNodeAllocationStrategy.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/allocation/AbstractNodeAllocationStrategy.java index 62c438fcc691..9c8ebd35aac5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/allocation/AbstractNodeAllocationStrategy.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/allocation/AbstractNodeAllocationStrategy.java @@ -33,6 +33,7 @@ import java.io.File; import java.io.FileNotFoundException; +import java.io.IOException; import java.util.Arrays; public abstract class AbstractNodeAllocationStrategy implements NodeAllocationStrategy { @@ -75,6 +76,9 @@ protected IWALNode createWALNode(String identifier, String folder) { } catch (FileNotFoundException e) { logger.error("Fail to create wal node", e); return WALFakeNode.getFailureInstance(e); + } catch (IOException e) { + logger.error("Meet exception when creating wal node", e); + return WALFakeNode.getFailureInstance(e); } } @@ -85,6 +89,9 @@ protected IWALNode createWALNode( } catch (FileNotFoundException e) { logger.error("Fail to create wal node", e); return WALFakeNode.getFailureInstance(e); + } catch (IOException e) { + logger.error("Meet exception when creating wal node", e); + return WALFakeNode.getFailureInstance(e); } } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/AbstractWALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/AbstractWALBuffer.java index 08bca5f6649f..c4c5bc13a3d4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/AbstractWALBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/AbstractWALBuffer.java @@ -29,7 +29,6 @@ import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.nio.file.Files; import java.nio.file.StandardCopyOption; @@ -58,7 +57,7 @@ public abstract class AbstractWALBuffer implements IWALBuffer { protected AbstractWALBuffer( String identifier, String logDirectory, long startFileVersion, long startSearchIndex) - throws FileNotFoundException { + throws IOException { this.identifier = identifier; this.logDirectory = logDirectory; File logDirFile = SystemFileFactory.INSTANCE.getFile(logDirectory); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java index 7bf33a93f40c..cd3696919a48 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java @@ -42,7 +42,6 @@ import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.MappedByteBuffer; @@ -119,7 +118,7 @@ public class WALBuffer extends AbstractWALBuffer { // manage wal files which have MemTableIds private final Map> memTableIdsOfWal = new ConcurrentHashMap<>(); - public WALBuffer(String identifier, String logDirectory) throws FileNotFoundException { + public WALBuffer(String identifier, String logDirectory) throws IOException { this(identifier, logDirectory, new CheckpointManager(identifier, logDirectory), 0, 0L); } @@ -129,7 +128,7 @@ public WALBuffer( CheckpointManager checkpointManager, long startFileVersion, long startSearchIndex) - throws FileNotFoundException { + throws IOException { super(identifier, logDirectory, startFileVersion, startSearchIndex); this.checkpointManager = checkpointManager; currentFileStatus = WALFileStatus.CONTAINS_NONE_SEARCH_INDEX; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/Checkpoint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/Checkpoint.java index f62d2a769e11..509b4142c5a9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/Checkpoint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/Checkpoint.java @@ -28,6 +28,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.concurrent.atomic.AtomicInteger; /** * Checkpoint is the basic element of .checkpoint file, including type, number of memTables, and @@ -69,6 +70,8 @@ public void serialize(ByteBuffer buffer) { } } + private static AtomicInteger count = new AtomicInteger(0); + public static Checkpoint deserialize(DataInputStream stream) throws IOException { byte typeNum = stream.readByte(); CheckpointType type = CheckpointType.valueOf(typeNum); @@ -76,6 +79,7 @@ public static Checkpoint deserialize(DataInputStream stream) throws IOException throw new IOException("unrecognized checkpoint type " + typeNum); } int cnt = stream.readInt(); + count.addAndGet(1); List memTableInfos = new ArrayList<>(cnt); for (int i = 0; i < cnt; ++i) { MemTableInfo memTableInfo = MemTableInfo.deserialize(stream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java index 8359330754d2..9de6fb044a9b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java @@ -34,7 +34,6 @@ import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Files; @@ -78,7 +77,7 @@ public class CheckpointManager implements AutoCloseable { // endregion - public CheckpointManager(String identifier, String logDirectory) throws FileNotFoundException { + public CheckpointManager(String identifier, String logDirectory) throws IOException { this.identifier = identifier; this.logDirectory = logDirectory; File logDirFile = SystemFileFactory.INSTANCE.getFile(logDirectory); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointReader.java index 081b3ed4a4fd..cc9651c50c8a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointReader.java @@ -47,7 +47,7 @@ public CheckpointReader(File logFile) { private void init() { checkpoints = new ArrayList<>(); try (DataInputStream logStream = - new DataInputStream(new BufferedInputStream(new WALInputStream(logFile)))) { + new DataInputStream(new WALInputStream(logFile))) { maxMemTableId = logStream.readLong(); while (logStream.available() > 0) { Checkpoint checkpoint = Checkpoint.deserialize(logStream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointWriter.java index cd6af4963776..322aa3c9f5d3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointWriter.java @@ -22,11 +22,11 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.checkpoint.Checkpoint; import java.io.File; -import java.io.FileNotFoundException; +import java.io.IOException; /** CheckpointWriter writes the binary {@link Checkpoint} into .checkpoint file. */ public class CheckpointWriter extends LogWriter { - public CheckpointWriter(File logFile) throws FileNotFoundException { + public CheckpointWriter(File logFile) throws IOException { super(logFile); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index dd83d48f5d7f..8b790388120a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -29,7 +29,6 @@ import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.nio.ByteBuffer; @@ -54,17 +53,20 @@ public abstract class LogWriter implements ILogWriter { IoTDBDescriptor.getInstance().getConfig().getWALCompressionAlgorithm(); private final ICompressor compressor = ICompressor.getCompressor(compressionAlg); private final ByteBuffer compressedByteBuffer; - private static final long MIN_COMPRESS_SIZE = 1024 * 512; + private static final long MIN_COMPRESS_SIZE = 0; private static AtomicLong WALTotalSize = new AtomicLong(0); private static AtomicLong lastReportTime = new AtomicLong(0); - protected LogWriter(File logFile) throws FileNotFoundException { + protected LogWriter(File logFile) throws IOException { this.logFile = logFile; this.logStream = new FileOutputStream(logFile, true); this.logChannel = this.logStream.getChannel(); + if (!logFile.exists() || logFile.length() == 0) { + this.logChannel.write(ByteBuffer.wrap(WALWriter.MAGIC_STRING.getBytes())); + } if (compressionAlg != CompressionType.UNCOMPRESSED) { compressedByteBuffer = - ByteBuffer.allocateDirect( + ByteBuffer.allocate( compressor.getMaxBytesForCompression( IoTDBDescriptor.getInstance().getConfig().getWalBufferSize())); } else { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java index 33287a511a32..f632684bcf16 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java @@ -18,8 +18,6 @@ */ package org.apache.iotdb.db.storageengine.dataregion.wal.io; -import org.apache.iotdb.db.conf.IoTDBDescriptor; - import org.apache.tsfile.compress.IUnCompressor; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.slf4j.Logger; @@ -30,6 +28,7 @@ import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import java.util.Arrays; import java.util.Objects; public class WALInputStream extends InputStream implements AutoCloseable { @@ -38,9 +37,8 @@ public class WALInputStream extends InputStream implements AutoCloseable { private final FileChannel channel; private final ByteBuffer headerBuffer = ByteBuffer.allocate(Integer.BYTES + 1); private final ByteBuffer compressedHeader = ByteBuffer.allocate(Integer.BYTES); - private ByteBuffer dataBuffer = - ByteBuffer.allocate( - IoTDBDescriptor.getInstance().getConfig().getWalBufferSize()); // uncompressed data buffer + private ByteBuffer dataBuffer = null; + File logFile; enum FileVersion { V1, @@ -53,30 +51,59 @@ enum FileVersion { public WALInputStream(File logFile) throws IOException { channel = FileChannel.open(logFile.toPath()); analyzeFileVersion(); + this.logFile = logFile; } private void analyzeFileVersion() throws IOException { - ByteBuffer magicStringBytes = ByteBuffer.allocate(WALWriter.MAGIC_STRING_BYTES); - channel.read(magicStringBytes, channel.size() - WALWriter.MAGIC_STRING_BYTES); - magicStringBytes.flip(); - String magicString = new String(magicStringBytes.array()); - if (magicString.equals(WALWriter.MAGIC_STRING)) { - version = FileVersion.V2; - } else if (magicString.startsWith(WALWriter.MAGIC_STRING_V1)) { - version = FileVersion.V1; - } else { + if (channel.size() < WALWriter.MAGIC_STRING_BYTES) { version = FileVersion.UNKNOWN; + return; } + if (isCurrentVersion()) { + this.version = FileVersion.V2; + return; + } + this.version = FileVersion.V1; + } + + private boolean isCurrentVersion() throws IOException { + channel.position(0); + ByteBuffer buffer = ByteBuffer.allocate(WALWriter.MAGIC_STRING_BYTES); + channel.read(buffer); + return new String(buffer.array()).equals(WALWriter.MAGIC_STRING); } @Override public int read() throws IOException { - if (Objects.isNull(dataBuffer) || dataBuffer.position() == dataBuffer.limit()) { + if (Objects.isNull(dataBuffer) || dataBuffer.position() >= dataBuffer.limit()) { loadNextSegment(); } return dataBuffer.get() & 0xFF; } + @Override + public int read(byte b[], int off, int len) throws IOException { + if (Objects.isNull(dataBuffer) || dataBuffer.position() >= dataBuffer.limit()) { + loadNextSegment(); + } + if (dataBuffer.remaining() >= len) { + dataBuffer.get(b, off, len); + return len; + } + int toBeRead = len; + while (toBeRead > 0) { + int remaining = dataBuffer.remaining(); + int bytesRead = Math.min(remaining, toBeRead); + dataBuffer.get(b, off, bytesRead); + off += bytesRead; + toBeRead -= bytesRead; + if (toBeRead > 0) { + loadNextSegment(); + } + } + return len; + } + @Override public void close() throws IOException { channel.close(); @@ -120,16 +147,13 @@ private void loadNextSegmentV2() throws IOException { } compressedHeader.flip(); int uncompressedSize = compressedHeader.getInt(); - if (uncompressedSize > dataBuffer.capacity()) { - // enlarge buffer - dataBuffer = ByteBuffer.allocateDirect(uncompressedSize); - } - ByteBuffer compressedData = ByteBuffer.allocate(dataBufferSize); + dataBuffer = ByteBuffer.allocateDirect(uncompressedSize); + ByteBuffer compressedData = ByteBuffer.allocateDirect(dataBufferSize); if (channel.read(compressedData) != dataBufferSize) { throw new IOException("Unexpected end of file"); } compressedData.flip(); - IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.GZIP); + IUnCompressor unCompressor = IUnCompressor.getUnCompressor(compressionType); dataBuffer.clear(); unCompressor.uncompress(compressedData, dataBuffer); } else { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java index 5f7a8cf6ff8b..6aa3331c8ffd 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java @@ -22,11 +22,15 @@ import org.apache.iotdb.consensus.iot.log.ConsensusReqReader; import org.apache.iotdb.db.utils.SerializedSize; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -36,6 +40,7 @@ * entry and the number of entries. */ public class WALMetaData implements SerializedSize { + private static final Logger logger = LoggerFactory.getLogger(WALMetaData.class); // search index 8 byte, wal entries' number 4 bytes private static final int FIXED_SERIALIZED_SIZE = Long.BYTES + Integer.BYTES; @@ -79,12 +84,13 @@ public int serializedSize() { + (memTablesId.isEmpty() ? 0 : Integer.BYTES + memTablesId.size() * Long.BYTES); } - public void serialize(ByteBuffer buffer) { + public void serialize(File file, ByteBuffer buffer) { buffer.putLong(firstSearchIndex); buffer.putInt(buffersSize.size()); for (int size : buffersSize) { buffer.putInt(size); } + logger.info("{} Buffer size is {}", file.getAbsolutePath(), buffersSize); if (!memTablesId.isEmpty()) { buffer.putInt(memTablesId.size()); for (long memTableId : memTablesId) { @@ -93,13 +99,14 @@ public void serialize(ByteBuffer buffer) { } } - public static WALMetaData deserialize(ByteBuffer buffer) { + public static WALMetaData deserialize(File file, ByteBuffer buffer) { long firstSearchIndex = buffer.getLong(); int entriesNum = buffer.getInt(); List buffersSize = new ArrayList<>(entriesNum); for (int i = 0; i < entriesNum; ++i) { buffersSize.add(buffer.getInt()); } + logger.info("{} buffer size is {}", file.getAbsolutePath(), buffersSize); Set memTablesId = new HashSet<>(); if (buffer.hasRemaining()) { int memTablesIdNum = buffer.getInt(); @@ -123,8 +130,12 @@ public long getFirstSearchIndex() { } public static WALMetaData readFromWALFile(File logFile, FileChannel channel) throws IOException { - if (channel.size() < WALWriter.MAGIC_STRING_BYTES || !isValidMagicString(channel)) { - throw new IOException(String.format("Broken wal file %s", logFile)); + if (channel.size() < WALWriter.MAGIC_STRING_BYTES) { + throw new IOException( + String.format("Broken wal file %s, size is %d", logFile, channel.size())); + } + if (!isValidMagicString(logFile, channel)) { + throw new IOException(String.format("Broken wal file %s, magic string invalid", logFile)); } // load metadata size ByteBuffer metadataSizeBuf = ByteBuffer.allocate(Integer.BYTES); @@ -133,10 +144,13 @@ public static WALMetaData readFromWALFile(File logFile, FileChannel channel) thr metadataSizeBuf.flip(); // load metadata int metadataSize = metadataSizeBuf.getInt(); + logger.error( + "{} metadataSize: {}, position {}", logFile.getAbsolutePath(), metadataSize, position); ByteBuffer metadataBuf = ByteBuffer.allocate(metadataSize); channel.read(metadataBuf, position - metadataSize); metadataBuf.flip(); - WALMetaData metaData = WALMetaData.deserialize(metadataBuf); + logger.error("{} metadata {}", logFile.getAbsolutePath(), Arrays.toString(metadataBuf.array())); + WALMetaData metaData = WALMetaData.deserialize(logFile, metadataBuf); // versions before V1.3, should recover memTable ids from entries if (metaData.memTablesId.isEmpty()) { int offset = Byte.BYTES; @@ -152,11 +166,12 @@ public static WALMetaData readFromWALFile(File logFile, FileChannel channel) thr return metaData; } - private static boolean isValidMagicString(FileChannel channel) throws IOException { + private static boolean isValidMagicString(File logFile, FileChannel channel) throws IOException { ByteBuffer magicStringBytes = ByteBuffer.allocate(WALWriter.MAGIC_STRING_BYTES); channel.read(magicStringBytes, channel.size() - WALWriter.MAGIC_STRING_BYTES); magicStringBytes.flip(); String magicString = new String(magicStringBytes.array()); + logger.error("{} Magic string {}", logFile.getAbsolutePath(), magicString); return magicString.equals(WALWriter.MAGIC_STRING) || magicString.startsWith(WALWriter.MAGIC_STRING_V1); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java index d922aae2e991..b436b1f37bcf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java @@ -24,13 +24,17 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALSignalEntry; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileStatus; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; /** WALWriter writes the binary {@link WALEntry} into .wal file. */ public class WALWriter extends LogWriter { + private static final Logger logger = LoggerFactory.getLogger(WALWriter.class); public static final String MAGIC_STRING_V1 = "WAL"; public static final String MAGIC_STRING = "V2-WAL"; public static final int MAGIC_STRING_BYTES = MAGIC_STRING.getBytes().length; @@ -39,7 +43,7 @@ public class WALWriter extends LogWriter { // wal files' metadata protected final WALMetaData metaData = new WALMetaData(); - public WALWriter(File logFile) throws FileNotFoundException { + public WALWriter(File logFile) throws IOException { super(logFile); } @@ -69,14 +73,26 @@ private void endFile() throws IOException { // mark info part ends endMarker.serialize(buffer); // flush meta data - metaData.serialize(buffer); + metaData.serialize(logFile, buffer); + logger.info("{} metadata {}", logFile.getAbsolutePath(), Arrays.toString(buffer.array())); buffer.putInt(metaDataSize); // add magic string buffer.put(MAGIC_STRING.getBytes()); - write(buffer); + logger.error( + "{} metadata size {}, position {}", + logFile.getAbsolutePath(), + metaDataSize, + logChannel.position()); + writeMetadata(buffer); this.isEndFile = false; } + private void writeMetadata(ByteBuffer buffer) throws IOException { + size += buffer.position(); + buffer.flip(); + logChannel.write(buffer); + } + @Override public void close() throws IOException { endFile(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java index e4a497de4d9b..bcf7303eadeb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java @@ -64,6 +64,7 @@ import java.io.File; import java.io.FileNotFoundException; +import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; @@ -107,13 +108,13 @@ public class WALNode implements IWALNode { // insert nodes whose search index are before this value can be deleted safely private volatile long safelyDeletedSearchIndex = DEFAULT_SAFELY_DELETED_SEARCH_INDEX; - public WALNode(String identifier, String logDirectory) throws FileNotFoundException { + public WALNode(String identifier, String logDirectory) throws IOException { this(identifier, logDirectory, 0, 0L); } public WALNode( String identifier, String logDirectory, long startFileVersion, long startSearchIndex) - throws FileNotFoundException { + throws IOException { this.identifier = identifier; this.logDirectory = SystemFileFactory.INSTANCE.getFile(logDirectory); if (!this.logDirectory.exists() && this.logDirectory.mkdirs()) { @@ -221,7 +222,7 @@ public void unpinMemTable(long memTableId) throws MemTablePinException { /** Delete outdated .wal files. */ public void deleteOutdatedFiles() { try { - new DeleteOutdatedFileTask().run(); + // new DeleteOutdatedFileTask().run(); } catch (Exception e) { logger.error("Fail to delete wal node-{}'s outdated files.", identifier, e); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriter.java index bec9979be94e..7015d45c58b0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriter.java @@ -28,8 +28,8 @@ import java.nio.channels.FileChannel; import java.nio.file.StandardOpenOption; +import static org.apache.iotdb.db.storageengine.dataregion.wal.io.WALWriter.MAGIC_STRING; import static org.apache.iotdb.db.storageengine.dataregion.wal.io.WALWriter.MAGIC_STRING_BYTES; -import static org.apache.iotdb.db.storageengine.dataregion.wal.io.WALWriter.MAGIC_STRING_V1; /** Check whether the wal file is broken and recover it. */ public class WALRecoverWriter { @@ -45,7 +45,7 @@ public void recover(WALMetaData metaData) throws IOException { if (logFile.length() < MAGIC_STRING_BYTES) { // file without magic string truncateSize = 0; } else { - if (readTailMagic().equals(MAGIC_STRING_V1)) { // complete file + if (readTailMagic().equals(MAGIC_STRING)) { // complete file return; } else { // file with broken magic string truncateSize = metaData.getBuffersSize().stream().mapToInt(Integer::intValue).sum(); From 95c0fd5ed192a88194909422ed4d89a48491cc35 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Sun, 19 May 2024 14:46:32 +0800 Subject: [PATCH 11/32] fix bug --- .../java/org/apache/iotdb/SessionExample.java | 62 +++++++++---------- .../dataregion/wal/io/CheckpointReader.java | 4 +- .../dataregion/wal/io/WALByteBufReader.java | 11 +++- .../dataregion/wal/io/WALInputStream.java | 25 +++++++- .../dataregion/wal/io/WALMetaData.java | 22 +++++-- .../wal/recover/WALNodeRecoverTask.java | 1 + 6 files changed, 83 insertions(+), 42 deletions(-) diff --git a/example/session/src/main/java/org/apache/iotdb/SessionExample.java b/example/session/src/main/java/org/apache/iotdb/SessionExample.java index beb81118ac28..ab807806024b 100644 --- a/example/session/src/main/java/org/apache/iotdb/SessionExample.java +++ b/example/session/src/main/java/org/apache/iotdb/SessionExample.java @@ -111,17 +111,17 @@ public static void main(String[] args) // deleteTimeseries(); // setTimeout(); -// sessionEnableRedirect = new Session(LOCAL_HOST, 6667, "root", "root"); -// sessionEnableRedirect.setEnableQueryRedirection(true); -// sessionEnableRedirect.open(false); -// -// // set session fetchSize -// sessionEnableRedirect.setFetchSize(10000); -// -// fastLastDataQueryForOneDevice(); -// insertRecord4Redirect(); -// query4Redirect(); -// sessionEnableRedirect.close(); + // sessionEnableRedirect = new Session(LOCAL_HOST, 6667, "root", "root"); + // sessionEnableRedirect.setEnableQueryRedirection(true); + // sessionEnableRedirect.open(false); + // + // // set session fetchSize + // sessionEnableRedirect.setFetchSize(10000); + // + // fastLastDataQueryForOneDevice(); + // insertRecord4Redirect(); + // query4Redirect(); + // sessionEnableRedirect.close(); session.close(); } @@ -423,26 +423,26 @@ private static void insertTablet() throws IoTDBConnectionException, StatementExe } // Method 2 to add tablet data -// long[] timestamps = tablet.timestamps; -// Object[] values = tablet.values; -// -// for (long time = 0; time < 100; time++) { -// int row = tablet.rowSize++; -// timestamps[row] = time; -// for (int i = 0; i < 3; i++) { -// long[] sensor = (long[]) values[i]; -// sensor[row] = i; -// } -// if (tablet.rowSize == tablet.getMaxRowNumber()) { -// session.insertTablet(tablet, true); -// tablet.reset(); -// } -// } -// -// if (tablet.rowSize != 0) { -// session.insertTablet(tablet); -// tablet.reset(); -// } + // long[] timestamps = tablet.timestamps; + // Object[] values = tablet.values; + // + // for (long time = 0; time < 100; time++) { + // int row = tablet.rowSize++; + // timestamps[row] = time; + // for (int i = 0; i < 3; i++) { + // long[] sensor = (long[]) values[i]; + // sensor[row] = i; + // } + // if (tablet.rowSize == tablet.getMaxRowNumber()) { + // session.insertTablet(tablet, true); + // tablet.reset(); + // } + // } + // + // if (tablet.rowSize != 0) { + // session.insertTablet(tablet); + // tablet.reset(); + // } } private static void insertTabletWithNullValues() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointReader.java index cc9651c50c8a..578ab21ae8c7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/CheckpointReader.java @@ -24,7 +24,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedInputStream; import java.io.DataInputStream; import java.io.File; import java.io.IOException; @@ -46,8 +45,7 @@ public CheckpointReader(File logFile) { private void init() { checkpoints = new ArrayList<>(); - try (DataInputStream logStream = - new DataInputStream(new WALInputStream(logFile))) { + try (DataInputStream logStream = new DataInputStream(new WALInputStream(logFile))) { maxMemTableId = logStream.readLong(); while (logStream.available() > 0) { Checkpoint checkpoint = Checkpoint.deserialize(logStream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java index ad3b7479de95..4d592d208267 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java @@ -21,6 +21,9 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.io.Closeable; import java.io.DataInputStream; import java.io.File; @@ -35,6 +38,7 @@ * {@link Iterator}. */ public class WALByteBufReader implements Closeable { + private static final Logger logger = LoggerFactory.getLogger(WALByteBufReader.class); private final File logFile; private final FileChannel channel; private final WALMetaData metaData; @@ -51,6 +55,7 @@ public WALByteBufReader(File logFile, FileChannel channel) throws IOException { this.logStream = new DataInputStream(new WALInputStream(logFile)); this.metaData = WALMetaData.readFromWALFile(logFile, channel); this.sizeIterator = metaData.getBuffersSize().iterator(); + logger.info("{} Buffer sizes is {}", logFile.getAbsolutePath(), metaData.getBuffersSize()); channel.position(0); } @@ -67,8 +72,12 @@ public boolean hasNext() { public ByteBuffer next() throws IOException { int size = sizeIterator.next(); ByteBuffer buffer = ByteBuffer.allocate(size); + /* + Notice, we don't need to flip the buffer after calling + logStream.readFully, since this function does not change + the position of the buffer. + */ logStream.readFully(buffer.array(), 0, size); - buffer.flip(); return buffer; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java index f632684bcf16..5e1825fa6d67 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java @@ -28,7 +28,6 @@ import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; -import java.util.Arrays; import java.util.Objects; public class WALInputStream extends InputStream implements AutoCloseable { @@ -82,23 +81,45 @@ public int read() throws IOException { } @Override - public int read(byte b[], int off, int len) throws IOException { + public int read(byte[] b, int off, int len) throws IOException { if (Objects.isNull(dataBuffer) || dataBuffer.position() >= dataBuffer.limit()) { loadNextSegment(); } + logger.error("Stack trace ", new Exception()); if (dataBuffer.remaining() >= len) { + int remain = dataBuffer.remaining(); dataBuffer.get(b, off, len); + logger.info( + "{} Read without loading, need {}, read {}", + logFile.getAbsolutePath(), + len, + remain - dataBuffer.remaining()); return len; } + logger.info( + "{} Read with loading, need {}, remain {}", + logFile.getAbsolutePath(), + len, + dataBuffer.remaining()); int toBeRead = len; while (toBeRead > 0) { int remaining = dataBuffer.remaining(); int bytesRead = Math.min(remaining, toBeRead); dataBuffer.get(b, off, bytesRead); + logger.info( + "{} Want to read {} bytes, actually read {}, remain {} to be read", + logFile.getAbsolutePath(), + bytesRead, + remaining - dataBuffer.remaining(), + toBeRead); off += bytesRead; toBeRead -= bytesRead; if (toBeRead > 0) { loadNextSegment(); + logger.info( + "{} Load next segment, segment size is {}", + logFile.getAbsolutePath(), + dataBuffer.limit()); } } return len; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java index 6aa3331c8ffd..11a39e6df49b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java @@ -85,18 +85,29 @@ public int serializedSize() { } public void serialize(File file, ByteBuffer buffer) { + ByteBuffer tmpBuffer = ByteBuffer.allocate(serializedSize()); buffer.putLong(firstSearchIndex); + tmpBuffer.putLong(firstSearchIndex); buffer.putInt(buffersSize.size()); + tmpBuffer.putInt(buffersSize.size()); for (int size : buffersSize) { buffer.putInt(size); + tmpBuffer.putInt(size); } - logger.info("{} Buffer size is {}", file.getAbsolutePath(), buffersSize); if (!memTablesId.isEmpty()) { buffer.putInt(memTablesId.size()); + tmpBuffer.putInt(memTablesId.size()); for (long memTableId : memTablesId) { buffer.putLong(memTableId); + tmpBuffer.putLong(memTableId); } } + tmpBuffer.flip(); + logger.info( + "{} Buffer size is {}, serialize is {}", + file.getAbsolutePath(), + buffersSize, + Arrays.toString(tmpBuffer.array())); } public static WALMetaData deserialize(File file, ByteBuffer buffer) { @@ -106,7 +117,11 @@ public static WALMetaData deserialize(File file, ByteBuffer buffer) { for (int i = 0; i < entriesNum; ++i) { buffersSize.add(buffer.getInt()); } - logger.info("{} buffer size is {}", file.getAbsolutePath(), buffersSize); + logger.info( + "{} recover buffer size is {}, buf is {}", + file.getAbsolutePath(), + buffersSize, + Arrays.toString(buffer.array())); Set memTablesId = new HashSet<>(); if (buffer.hasRemaining()) { int memTablesIdNum = buffer.getInt(); @@ -144,12 +159,9 @@ public static WALMetaData readFromWALFile(File logFile, FileChannel channel) thr metadataSizeBuf.flip(); // load metadata int metadataSize = metadataSizeBuf.getInt(); - logger.error( - "{} metadataSize: {}, position {}", logFile.getAbsolutePath(), metadataSize, position); ByteBuffer metadataBuf = ByteBuffer.allocate(metadataSize); channel.read(metadataBuf, position - metadataSize); metadataBuf.flip(); - logger.error("{} metadata {}", logFile.getAbsolutePath(), Arrays.toString(metadataBuf.array())); WALMetaData metaData = WALMetaData.deserialize(logFile, metadataBuf); // versions before V1.3, should recover memTable ids from entries if (metaData.memTablesId.isEmpty()) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java index 237b353c9f9a..d929c8e14f6a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java @@ -258,6 +258,7 @@ private void recoverTsFiles() { } while (reader.hasNext()) { ByteBuffer buffer = reader.next(); + logger.info("{} buffer size {}", walFile.getAbsolutePath(), buffer.limit()); // see WALInfoEntry#serialize, entry type buffer.position(Byte.BYTES); long memTableId = buffer.getLong(); From 23d1743772980fd6777c3218ace1b0983e10b02c Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Sun, 19 May 2024 14:59:26 +0800 Subject: [PATCH 12/32] remove useless log --- .../java/org/apache/iotdb/SessionExample.java | 64 +++++++++---------- .../confignode/conf/ConfigNodeConfig.java | 4 +- .../org/apache/iotdb/db/conf/IoTDBConfig.java | 4 +- .../dataregion/wal/checkpoint/Checkpoint.java | 4 -- .../dataregion/wal/io/LogWriter.java | 15 +---- .../dataregion/wal/io/WALByteBufReader.java | 5 -- .../dataregion/wal/io/WALInputStream.java | 22 ------- .../dataregion/wal/io/WALMetaData.java | 33 ++-------- .../dataregion/wal/io/WALWriter.java | 9 --- .../dataregion/wal/node/WALNode.java | 2 +- .../wal/recover/WALNodeRecoverTask.java | 1 - 11 files changed, 45 insertions(+), 118 deletions(-) diff --git a/example/session/src/main/java/org/apache/iotdb/SessionExample.java b/example/session/src/main/java/org/apache/iotdb/SessionExample.java index ab807806024b..0daec0885bf7 100644 --- a/example/session/src/main/java/org/apache/iotdb/SessionExample.java +++ b/example/session/src/main/java/org/apache/iotdb/SessionExample.java @@ -111,17 +111,17 @@ public static void main(String[] args) // deleteTimeseries(); // setTimeout(); - // sessionEnableRedirect = new Session(LOCAL_HOST, 6667, "root", "root"); - // sessionEnableRedirect.setEnableQueryRedirection(true); - // sessionEnableRedirect.open(false); - // - // // set session fetchSize - // sessionEnableRedirect.setFetchSize(10000); - // - // fastLastDataQueryForOneDevice(); - // insertRecord4Redirect(); - // query4Redirect(); - // sessionEnableRedirect.close(); + sessionEnableRedirect = new Session(LOCAL_HOST, 6667, "root", "root"); + sessionEnableRedirect.setEnableQueryRedirection(true); + sessionEnableRedirect.open(false); + + // set session fetchSize + sessionEnableRedirect.setFetchSize(10000); + + fastLastDataQueryForOneDevice(); + insertRecord4Redirect(); + query4Redirect(); + sessionEnableRedirect.close(); session.close(); } @@ -403,7 +403,7 @@ private static void insertTablet() throws IoTDBConnectionException, StatementExe // Method 1 to add tablet data long timestamp = System.currentTimeMillis(); - for (long row = 0; row < 200; row++) { + for (long row = 0; row < 100; row++) { int rowIndex = tablet.rowSize++; tablet.addTimestamp(rowIndex, timestamp); for (int s = 0; s < 3; s++) { @@ -423,26 +423,26 @@ private static void insertTablet() throws IoTDBConnectionException, StatementExe } // Method 2 to add tablet data - // long[] timestamps = tablet.timestamps; - // Object[] values = tablet.values; - // - // for (long time = 0; time < 100; time++) { - // int row = tablet.rowSize++; - // timestamps[row] = time; - // for (int i = 0; i < 3; i++) { - // long[] sensor = (long[]) values[i]; - // sensor[row] = i; - // } - // if (tablet.rowSize == tablet.getMaxRowNumber()) { - // session.insertTablet(tablet, true); - // tablet.reset(); - // } - // } - // - // if (tablet.rowSize != 0) { - // session.insertTablet(tablet); - // tablet.reset(); - // } + long[] timestamps = tablet.timestamps; + Object[] values = tablet.values; + + for (long time = 0; time < 100; time++) { + int row = tablet.rowSize++; + timestamps[row] = time; + for (int i = 0; i < 3; i++) { + long[] sensor = (long[]) values[i]; + sensor[row] = i; + } + if (tablet.rowSize == tablet.getMaxRowNumber()) { + session.insertTablet(tablet, true); + tablet.reset(); + } + } + + if (tablet.rowSize != 0) { + session.insertTablet(tablet); + tablet.reset(); + } } private static void insertTabletWithNullValues() diff --git a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java index 6a7f91cf3547..603892e36811 100644 --- a/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java +++ b/iotdb-core/confignode/src/main/java/org/apache/iotdb/confignode/conf/ConfigNodeConfig.java @@ -94,14 +94,14 @@ public class ConfigNodeConfig { /** The policy of extension DataRegionGroup for each Database. */ private RegionGroupExtensionPolicy dataRegionGroupExtensionPolicy = - RegionGroupExtensionPolicy.CUSTOM; + RegionGroupExtensionPolicy.AUTO; /** * When set data_region_group_extension_policy=CUSTOM, this parameter is the default number of * DataRegionGroups for each Database. When set data_region_group_extension_policy=AUTO, this * parameter is the default minimal number of DataRegionGroups for each Database. */ - private int defaultDataRegionGroupNumPerDatabase = 1; + private int defaultDataRegionGroupNumPerDatabase = 2; /** The maximum number of DataRegions expected to be managed by each DataNode. */ private double dataRegionPerDataNode = 5.0; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index f8746ae0cecf..186ca64229a9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -215,7 +215,7 @@ public class IoTDBConfig { private volatile long walSyncModeFsyncDelayInMs = 3; /** Buffer size of each wal node. Unit: byte */ - private int walBufferSize = 4 * 1024; + private int walBufferSize = 32 * 1024 * 1024; /** max total direct buffer off heap memory size proportion */ private double maxDirectBufferOffHeapMemorySizeProportion = 0.8; @@ -224,7 +224,7 @@ public class IoTDBConfig { private int walBufferQueueCapacity = 500; /** Size threshold of each wal file. Unit: byte */ - private volatile long walFileSizeThresholdInByte = 4 * 1024L; + private volatile long walFileSizeThresholdInByte = 30 * 1024 * 1024L; /** Size threshold of each checkpoint file. Unit: byte */ private volatile long checkpointFileSizeThresholdInByte = 3 * 1024 * 1024L; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/Checkpoint.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/Checkpoint.java index 509b4142c5a9..f62d2a769e11 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/Checkpoint.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/Checkpoint.java @@ -28,7 +28,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; -import java.util.concurrent.atomic.AtomicInteger; /** * Checkpoint is the basic element of .checkpoint file, including type, number of memTables, and @@ -70,8 +69,6 @@ public void serialize(ByteBuffer buffer) { } } - private static AtomicInteger count = new AtomicInteger(0); - public static Checkpoint deserialize(DataInputStream stream) throws IOException { byte typeNum = stream.readByte(); CheckpointType type = CheckpointType.valueOf(typeNum); @@ -79,7 +76,6 @@ public static Checkpoint deserialize(DataInputStream stream) throws IOException throw new IOException("unrecognized checkpoint type " + typeNum); } int cnt = stream.readInt(); - count.addAndGet(1); List memTableInfos = new ArrayList<>(cnt); for (int i = 0; i < cnt; ++i) { MemTableInfo memTableInfo = MemTableInfo.deserialize(stream); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index 8b790388120a..021611920621 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -34,7 +34,6 @@ import java.nio.ByteBuffer; import java.nio.channels.ClosedChannelException; import java.nio.channels.FileChannel; -import java.util.concurrent.atomic.AtomicLong; /** * LogWriter writes the binary logs into a file, including writing {@link WALEntry} into .wal file @@ -47,15 +46,13 @@ public abstract class LogWriter implements ILogWriter { protected final FileOutputStream logStream; protected final FileChannel logChannel; protected long size; - protected boolean isEndFile = false; private final ByteBuffer headerBuffer = ByteBuffer.allocate(Integer.BYTES * 2 + 1); private static final CompressionType compressionAlg = IoTDBDescriptor.getInstance().getConfig().getWALCompressionAlgorithm(); private final ICompressor compressor = ICompressor.getCompressor(compressionAlg); private final ByteBuffer compressedByteBuffer; - private static final long MIN_COMPRESS_SIZE = 0; - private static AtomicLong WALTotalSize = new AtomicLong(0); - private static AtomicLong lastReportTime = new AtomicLong(0); + // Minimum size to compress, default is 32 KB + private static final long MIN_COMPRESS_SIZE = 32 * 1024; protected LogWriter(File logFile) throws IOException { this.logFile = logFile; @@ -80,8 +77,7 @@ public void write(ByteBuffer buffer) throws IOException { buffer.flip(); boolean compressed = false; int uncompressedSize = bufferSize; - if (!isEndFile - && compressionAlg != CompressionType.UNCOMPRESSED + if (compressionAlg != CompressionType.UNCOMPRESSED && bufferSize > MIN_COMPRESS_SIZE /* Do not compress buffer that is less than 512KB */) { compressedByteBuffer.clear(); compressor.compress(buffer, compressedByteBuffer); @@ -110,14 +106,9 @@ public void write(ByteBuffer buffer) throws IOException { long before = logChannel.position(); logChannel.write(headerBuffer); logChannel.write(buffer); - WALTotalSize.addAndGet(logChannel.position() - before); } catch (ClosedChannelException e) { logger.warn("Cannot write to {}", logFile, e); } - if (System.currentTimeMillis() - lastReportTime.get() > 10_000L) { - lastReportTime.set(System.currentTimeMillis()); - logger.info("WAL total size is {} GB", WALTotalSize.doubleValue() / 1024 / 1024 / 1024); - } } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java index 4d592d208267..3e0909d6c52e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java @@ -21,9 +21,6 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.Closeable; import java.io.DataInputStream; import java.io.File; @@ -38,7 +35,6 @@ * {@link Iterator}. */ public class WALByteBufReader implements Closeable { - private static final Logger logger = LoggerFactory.getLogger(WALByteBufReader.class); private final File logFile; private final FileChannel channel; private final WALMetaData metaData; @@ -55,7 +51,6 @@ public WALByteBufReader(File logFile, FileChannel channel) throws IOException { this.logStream = new DataInputStream(new WALInputStream(logFile)); this.metaData = WALMetaData.readFromWALFile(logFile, channel); this.sizeIterator = metaData.getBuffersSize().iterator(); - logger.info("{} Buffer sizes is {}", logFile.getAbsolutePath(), metaData.getBuffersSize()); channel.position(0); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java index 5e1825fa6d67..ae0fbb06a4ec 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java @@ -85,41 +85,19 @@ public int read(byte[] b, int off, int len) throws IOException { if (Objects.isNull(dataBuffer) || dataBuffer.position() >= dataBuffer.limit()) { loadNextSegment(); } - logger.error("Stack trace ", new Exception()); if (dataBuffer.remaining() >= len) { - int remain = dataBuffer.remaining(); dataBuffer.get(b, off, len); - logger.info( - "{} Read without loading, need {}, read {}", - logFile.getAbsolutePath(), - len, - remain - dataBuffer.remaining()); return len; } - logger.info( - "{} Read with loading, need {}, remain {}", - logFile.getAbsolutePath(), - len, - dataBuffer.remaining()); int toBeRead = len; while (toBeRead > 0) { int remaining = dataBuffer.remaining(); int bytesRead = Math.min(remaining, toBeRead); dataBuffer.get(b, off, bytesRead); - logger.info( - "{} Want to read {} bytes, actually read {}, remain {} to be read", - logFile.getAbsolutePath(), - bytesRead, - remaining - dataBuffer.remaining(), - toBeRead); off += bytesRead; toBeRead -= bytesRead; if (toBeRead > 0) { loadNextSegment(); - logger.info( - "{} Load next segment, segment size is {}", - logFile.getAbsolutePath(), - dataBuffer.limit()); } } return len; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java index 11a39e6df49b..7916af9308e5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java @@ -30,7 +30,6 @@ import java.nio.ByteBuffer; import java.nio.channels.FileChannel; import java.util.ArrayList; -import java.util.Arrays; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -85,43 +84,26 @@ public int serializedSize() { } public void serialize(File file, ByteBuffer buffer) { - ByteBuffer tmpBuffer = ByteBuffer.allocate(serializedSize()); buffer.putLong(firstSearchIndex); - tmpBuffer.putLong(firstSearchIndex); buffer.putInt(buffersSize.size()); - tmpBuffer.putInt(buffersSize.size()); for (int size : buffersSize) { buffer.putInt(size); - tmpBuffer.putInt(size); } if (!memTablesId.isEmpty()) { buffer.putInt(memTablesId.size()); - tmpBuffer.putInt(memTablesId.size()); for (long memTableId : memTablesId) { buffer.putLong(memTableId); - tmpBuffer.putLong(memTableId); } } - tmpBuffer.flip(); - logger.info( - "{} Buffer size is {}, serialize is {}", - file.getAbsolutePath(), - buffersSize, - Arrays.toString(tmpBuffer.array())); } - public static WALMetaData deserialize(File file, ByteBuffer buffer) { + public static WALMetaData deserialize(ByteBuffer buffer) { long firstSearchIndex = buffer.getLong(); int entriesNum = buffer.getInt(); List buffersSize = new ArrayList<>(entriesNum); for (int i = 0; i < entriesNum; ++i) { buffersSize.add(buffer.getInt()); } - logger.info( - "{} recover buffer size is {}, buf is {}", - file.getAbsolutePath(), - buffersSize, - Arrays.toString(buffer.array())); Set memTablesId = new HashSet<>(); if (buffer.hasRemaining()) { int memTablesIdNum = buffer.getInt(); @@ -145,12 +127,8 @@ public long getFirstSearchIndex() { } public static WALMetaData readFromWALFile(File logFile, FileChannel channel) throws IOException { - if (channel.size() < WALWriter.MAGIC_STRING_BYTES) { - throw new IOException( - String.format("Broken wal file %s, size is %d", logFile, channel.size())); - } - if (!isValidMagicString(logFile, channel)) { - throw new IOException(String.format("Broken wal file %s, magic string invalid", logFile)); + if (channel.size() < WALWriter.MAGIC_STRING_BYTES || !isValidMagicString(channel)) { + throw new IOException(String.format("Broken wal file %s", logFile)); } // load metadata size ByteBuffer metadataSizeBuf = ByteBuffer.allocate(Integer.BYTES); @@ -162,7 +140,7 @@ public static WALMetaData readFromWALFile(File logFile, FileChannel channel) thr ByteBuffer metadataBuf = ByteBuffer.allocate(metadataSize); channel.read(metadataBuf, position - metadataSize); metadataBuf.flip(); - WALMetaData metaData = WALMetaData.deserialize(logFile, metadataBuf); + WALMetaData metaData = WALMetaData.deserialize(metadataBuf); // versions before V1.3, should recover memTable ids from entries if (metaData.memTablesId.isEmpty()) { int offset = Byte.BYTES; @@ -178,12 +156,11 @@ public static WALMetaData readFromWALFile(File logFile, FileChannel channel) thr return metaData; } - private static boolean isValidMagicString(File logFile, FileChannel channel) throws IOException { + private static boolean isValidMagicString(FileChannel channel) throws IOException { ByteBuffer magicStringBytes = ByteBuffer.allocate(WALWriter.MAGIC_STRING_BYTES); channel.read(magicStringBytes, channel.size() - WALWriter.MAGIC_STRING_BYTES); magicStringBytes.flip(); String magicString = new String(magicStringBytes.array()); - logger.error("{} Magic string {}", logFile.getAbsolutePath(), magicString); return magicString.equals(WALWriter.MAGIC_STRING) || magicString.startsWith(WALWriter.MAGIC_STRING_V1); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java index b436b1f37bcf..95508e7186e7 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java @@ -30,7 +30,6 @@ import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; -import java.util.Arrays; /** WALWriter writes the binary {@link WALEntry} into .wal file. */ public class WALWriter extends LogWriter { @@ -64,7 +63,6 @@ public void updateMetaData(WALMetaData metaData) { } private void endFile() throws IOException { - this.isEndFile = true; WALSignalEntry endMarker = new WALSignalEntry(WALEntryType.WAL_FILE_INFO_END_MARKER); int metaDataSize = metaData.serializedSize(); ByteBuffer buffer = @@ -74,17 +72,10 @@ private void endFile() throws IOException { endMarker.serialize(buffer); // flush meta data metaData.serialize(logFile, buffer); - logger.info("{} metadata {}", logFile.getAbsolutePath(), Arrays.toString(buffer.array())); buffer.putInt(metaDataSize); // add magic string buffer.put(MAGIC_STRING.getBytes()); - logger.error( - "{} metadata size {}, position {}", - logFile.getAbsolutePath(), - metaDataSize, - logChannel.position()); writeMetadata(buffer); - this.isEndFile = false; } private void writeMetadata(ByteBuffer buffer) throws IOException { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java index bcf7303eadeb..9a4b99f9ec05 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNode.java @@ -222,7 +222,7 @@ public void unpinMemTable(long memTableId) throws MemTablePinException { /** Delete outdated .wal files. */ public void deleteOutdatedFiles() { try { - // new DeleteOutdatedFileTask().run(); + new DeleteOutdatedFileTask().run(); } catch (Exception e) { logger.error("Fail to delete wal node-{}'s outdated files.", identifier, e); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java index d929c8e14f6a..237b353c9f9a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java @@ -258,7 +258,6 @@ private void recoverTsFiles() { } while (reader.hasNext()) { ByteBuffer buffer = reader.next(); - logger.info("{} buffer size {}", walFile.getAbsolutePath(), buffer.limit()); // see WALInfoEntry#serialize, entry type buffer.position(Byte.BYTES); long memTableId = buffer.getLong(); From 8bedc72c3ed2d58a15795a18ef1612d1e0c99003 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Mon, 20 May 2024 14:38:41 +0800 Subject: [PATCH 13/32] remove one configuration --- .../src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 1 - 1 file changed, 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 186ca64229a9..f4f841d00b68 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -96,7 +96,6 @@ public class IoTDBConfig { "([" + PATH_SEPARATOR + "])?" + NODE_NAME_MATCHER + "(" + PARTIAL_NODE_MATCHER + ")*"; public static final Pattern NODE_PATTERN = Pattern.compile(NODE_MATCHER); - boolean enableWALCompression = true; /** Whether to enable the mqtt service. */ private boolean enableMQTTService = false; From 69ebc898dd29b8ebf4931b1e6269deaf6feba62b Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Fri, 24 May 2024 14:04:22 +0800 Subject: [PATCH 14/32] use compression rate to update wal disk usage --- .../db/storageengine/dataregion/wal/buffer/WALBuffer.java | 5 +++-- .../dataregion/wal/checkpoint/CheckpointManager.java | 5 +++-- .../db/storageengine/dataregion/wal/io/ILogWriter.java | 3 ++- .../db/storageengine/dataregion/wal/io/LogWriter.java | 7 ++++--- .../db/storageengine/dataregion/wal/io/WALWriter.java | 4 ++-- 5 files changed, 14 insertions(+), 10 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java index cd3696919a48..5096caf6783e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java @@ -520,8 +520,9 @@ public void run() { forceFlag, syncingBuffer.position(), syncingBuffer.capacity(), usedRatio * 100); // flush buffer to os + double compressionRate = 1.0; try { - currentWALFileWriter.write(syncingBuffer, info.metaData); + compressionRate = currentWALFileWriter.write(syncingBuffer, info.metaData); } catch (Throwable e) { logger.error( "Fail to sync wal node-{}'s buffer, change system mode to error.", identifier, e); @@ -534,7 +535,7 @@ public void run() { memTableIdsOfWal .computeIfAbsent(currentWALFileVersion, memTableIds -> new HashSet<>()) .addAll(info.metaData.getMemTablesId()); - checkpointManager.updateCostOfActiveMemTables(info.memTableId2WalDiskUsage); + checkpointManager.updateCostOfActiveMemTables(info.memTableId2WalDiskUsage, compressionRate); boolean forceSuccess = false; // try to roll log writer diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java index 9de6fb044a9b..412dfcfc0a0c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/checkpoint/CheckpointManager.java @@ -344,12 +344,13 @@ public long getFirstValidWALVersionId() { } /** Update wal disk cost of active memTables. */ - public void updateCostOfActiveMemTables(Map memTableId2WalDiskUsage) { + public void updateCostOfActiveMemTables( + Map memTableId2WalDiskUsage, double compressionRate) { for (Map.Entry memTableWalUsage : memTableId2WalDiskUsage.entrySet()) { memTableId2Info.computeIfPresent( memTableWalUsage.getKey(), (k, v) -> { - v.addWalDiskUsage(memTableWalUsage.getValue()); + v.addWalDiskUsage((long) (memTableWalUsage.getValue() * compressionRate)); return v; }); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/ILogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/ILogWriter.java index d89563a93281..f4d65612c470 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/ILogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/ILogWriter.java @@ -33,8 +33,9 @@ public interface ILogWriter extends Closeable { * * @param buffer content that have been converted to bytes * @throws IOException if an I/O error occurs + * @return Compression rate of the buffer after compression */ - void write(ByteBuffer buffer) throws IOException; + double write(ByteBuffer buffer) throws IOException; /** * Forces any updates to this file to be written to the storage device that contains it. diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index 021611920621..4ddebfac8ce0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -72,13 +72,14 @@ protected LogWriter(File logFile) throws IOException { } @Override - public void write(ByteBuffer buffer) throws IOException { + public double write(ByteBuffer buffer) throws IOException { int bufferSize = buffer.position(); buffer.flip(); boolean compressed = false; int uncompressedSize = bufferSize; if (compressionAlg != CompressionType.UNCOMPRESSED - && bufferSize > MIN_COMPRESS_SIZE /* Do not compress buffer that is less than 512KB */) { + /* Do not compress buffer that is less than min size */ + && bufferSize > MIN_COMPRESS_SIZE) { compressedByteBuffer.clear(); compressor.compress(buffer, compressedByteBuffer); buffer = compressedByteBuffer; @@ -103,12 +104,12 @@ public void write(ByteBuffer buffer) throws IOException { size += 5; try { headerBuffer.flip(); - long before = logChannel.position(); logChannel.write(headerBuffer); logChannel.write(buffer); } catch (ClosedChannelException e) { logger.warn("Cannot write to {}", logFile, e); } + return ((double) bufferSize / uncompressedSize); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java index 95508e7186e7..469dcaa8c698 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java @@ -51,11 +51,11 @@ public WALWriter(File logFile) throws IOException { * * @throws IOException when failing to write */ - public void write(ByteBuffer buffer, WALMetaData metaData) throws IOException { + public double write(ByteBuffer buffer, WALMetaData metaData) throws IOException { // update metadata updateMetaData(metaData); // flush buffer - write(buffer); + return write(buffer); } public void updateMetaData(WALMetaData metaData) { From d6ca95b5cc378b58bb323c2d3e2b49ec81865084 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Fri, 24 May 2024 22:30:45 +0800 Subject: [PATCH 15/32] fix ut --- .../dataregion/wal/io/LogWriter.java | 5 +- .../dataregion/wal/io/WALInputStream.java | 53 ++++++++++++++++++- .../wal/utils/WALEntryPosition.java | 29 ++++++++-- .../wal/recover/WALRecoverWriterTest.java | 13 +++-- 4 files changed, 91 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index 4ddebfac8ce0..b73b3b1e26f0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -52,7 +52,7 @@ public abstract class LogWriter implements ILogWriter { private final ICompressor compressor = ICompressor.getCompressor(compressionAlg); private final ByteBuffer compressedByteBuffer; // Minimum size to compress, default is 32 KB - private static final long MIN_COMPRESS_SIZE = 32 * 1024; + private static final long MIN_COMPRESS_SIZE = 0; protected LogWriter(File logFile) throws IOException { this.logFile = logFile; @@ -74,6 +74,9 @@ protected LogWriter(File logFile) throws IOException { @Override public double write(ByteBuffer buffer) throws IOException { int bufferSize = buffer.position(); + if (bufferSize == 0) { + return 1.0; + } buffer.flip(); boolean compressed = false; int uncompressedSize = bufferSize; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java index ae0fbb06a4ec..213e517d16ef 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java @@ -37,6 +37,7 @@ public class WALInputStream extends InputStream implements AutoCloseable { private final ByteBuffer headerBuffer = ByteBuffer.allocate(Integer.BYTES + 1); private final ByteBuffer compressedHeader = ByteBuffer.allocate(Integer.BYTES); private ByteBuffer dataBuffer = null; + private long fileSize; File logFile; enum FileVersion { @@ -49,6 +50,7 @@ enum FileVersion { public WALInputStream(File logFile) throws IOException { channel = FileChannel.open(logFile.toPath()); + fileSize = channel.size(); analyzeFileVersion(); this.logFile = logFile; } @@ -111,7 +113,11 @@ public void close() throws IOException { @Override public int available() throws IOException { - return (int) (channel.size() - channel.position()); + long size = (channel.size() - channel.position()); + if (!Objects.isNull(dataBuffer)) { + size += dataBuffer.limit() - dataBuffer.position(); + } + return (int) size; } private void loadNextSegment() throws IOException { @@ -126,6 +132,14 @@ private void loadNextSegment() throws IOException { private void loadNextSegmentV1() throws IOException { // just read raw data as input + if (channel.position() >= fileSize) { + throw new IOException("Unexpected end of file"); + } + if (Objects.isNull(dataBuffer)) { + // read 128 KB + dataBuffer = ByteBuffer.allocate(128 * 1024); + } + dataBuffer.clear(); channel.read(dataBuffer); dataBuffer.flip(); } @@ -180,4 +194,41 @@ private void tryLoadSegment() throws IOException { version = FileVersion.V1; } } + + public void skipToGivenPosition(long pos) throws IOException { + if (version == FileVersion.V2) { + channel.position(WALWriter.MAGIC_STRING_BYTES); + ByteBuffer buffer = ByteBuffer.allocate(Byte.BYTES + Integer.BYTES); + long posRemain = pos; + int currSegmentSize = 0; + while (posRemain > 0) { + buffer.clear(); + channel.read(buffer); + buffer.flip(); + buffer.get(); + currSegmentSize = buffer.getInt(); + if (posRemain >= currSegmentSize) { + posRemain -= currSegmentSize; + } else { + break; + } + } + dataBuffer = ByteBuffer.allocate(currSegmentSize); + channel.read(dataBuffer); + dataBuffer.position((int) posRemain); + } else { + dataBuffer.clear(); + channel.position(pos); + } + } + + public void read(ByteBuffer buffer) throws IOException { + int totalBytesToBeRead = buffer.remaining(); + int currReadBytes = Math.min(dataBuffer.remaining(), buffer.remaining()); + dataBuffer.get(buffer.array(), buffer.position(), currReadBytes); + if (totalBytesToBeRead - currReadBytes > 0) { + loadNextSegment(); + read(buffer); + } + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryPosition.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryPosition.java index 13b2f4c97041..1e89d8c546d0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryPosition.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryPosition.java @@ -21,6 +21,7 @@ import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertNode; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryValue; +import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALInputStream; import org.apache.iotdb.db.storageengine.dataregion.wal.node.WALNode; import org.apache.tsfile.utils.Pair; @@ -100,11 +101,11 @@ ByteBuffer read() throws IOException { if (!canRead()) { throw new IOException("Target file hasn't been specified."); } - try (FileChannel channel = openReadFileChannel()) { + try (WALInputStream is = openReadFileStream()) { + is.skipToGivenPosition(position); ByteBuffer buffer = ByteBuffer.allocate(size); - channel.position(position); - channel.read(buffer); - buffer.clear(); + is.read(buffer); + buffer.flip(); return buffer; } } @@ -135,6 +136,26 @@ public FileChannel openReadFileChannel() throws IOException { } } + public WALInputStream openReadFileStream() throws IOException { + if (isInSealedFile()) { + walFile = walNode.getWALFile(walFileVersionId); + return new WALInputStream(walFile); + } else { + try { + walFile = walNode.getWALFile(walFileVersionId); + return new WALInputStream(walFile); + } catch (IOException e) { + // unsealed file may be renamed after sealed, so we should try again + if (isInSealedFile()) { + walFile = walNode.getWALFile(walFileVersionId); + return new WALInputStream(walFile); + } else { + throw e; + } + } + } + } + public File getWalFile() { return walFile; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriterTest.java index c22505ad1a34..f492f015ec22 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriterTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriterTest.java @@ -71,9 +71,13 @@ public void testEmptyFile() throws IOException { // recover WALRecoverWriter walRecoverWriter = new WALRecoverWriter(logFile); walRecoverWriter.recover(walMetaData); - // verify file, marker + metadata(search index + size number) + metadata size + magic string + // verify file, marker + metadata(search index + size number) + metadata size + head magic + // string + tail magic string Assert.assertEquals( - Byte.BYTES + (Long.BYTES + Integer.BYTES) + Integer.BYTES + WALWriter.MAGIC_STRING_BYTES, + Byte.BYTES + + (Long.BYTES + Integer.BYTES) + + Integer.BYTES + + WALWriter.MAGIC_STRING_BYTES * 2, logFile.length()); try (WALByteBufReader reader = new WALByteBufReader(logFile)) { Assert.assertFalse(reader.hasNext()); @@ -95,7 +99,10 @@ public void testFileWithoutMagicString() throws IOException { walRecoverWriter.recover(walMetaData); // verify file, marker + metadata(search index + size number) + metadata size + magic string Assert.assertEquals( - Byte.BYTES + (Long.BYTES + Integer.BYTES) + Integer.BYTES + WALWriter.MAGIC_STRING_BYTES, + Byte.BYTES + + (Long.BYTES + Integer.BYTES) + + Integer.BYTES + + WALWriter.MAGIC_STRING_BYTES * 2, logFile.length()); try (WALByteBufReader reader = new WALByteBufReader(logFile)) { Assert.assertFalse(reader.hasNext()); From 4373559a1361bc77397c620a1b8e8b2449838ce0 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Sat, 25 May 2024 19:40:39 +0800 Subject: [PATCH 16/32] fix test --- .../dataregion/wal/io/WALInputStream.java | 47 ++++++++++++++++++- .../org/apache/iotdb/db/tools/WalChecker.java | 6 +-- 2 files changed, 48 insertions(+), 5 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java index 213e517d16ef..b0872ace299b 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java @@ -39,6 +39,7 @@ public class WALInputStream extends InputStream implements AutoCloseable { private ByteBuffer dataBuffer = null; private long fileSize; File logFile; + private long endOffset = -1; enum FileVersion { V1, @@ -52,9 +53,50 @@ public WALInputStream(File logFile) throws IOException { channel = FileChannel.open(logFile.toPath()); fileSize = channel.size(); analyzeFileVersion(); + getEndOffset(); this.logFile = logFile; } + private void getEndOffset() throws IOException { + if (channel.size() < WALWriter.MAGIC_STRING_BYTES + Integer.BYTES) { + endOffset = channel.size(); + return; + } + ByteBuffer metadataSizeBuf = ByteBuffer.allocate(Integer.BYTES); + long position; + try { + if (version == FileVersion.V2) { + ByteBuffer magicStringBuffer = ByteBuffer.allocate(WALWriter.MAGIC_STRING_BYTES); + channel.read(magicStringBuffer, channel.size() - WALWriter.MAGIC_STRING_BYTES); + magicStringBuffer.flip(); + if (!new String(magicStringBuffer.array()).equals(WALWriter.MAGIC_STRING)) { + // this is a broken wal file + endOffset = channel.size(); + return; + } + position = channel.size() - WALWriter.MAGIC_STRING_BYTES - Integer.BYTES; + } else { + ByteBuffer magicStringBuffer = + ByteBuffer.allocate(WALWriter.MAGIC_STRING_V1.getBytes().length); + channel.read( + magicStringBuffer, channel.size() - WALWriter.MAGIC_STRING_V1.getBytes().length); + magicStringBuffer.flip(); + if (!new String(magicStringBuffer.array()).equals(WALWriter.MAGIC_STRING_V1)) { + // this is a broken wal file + endOffset = channel.size(); + return; + } + position = channel.size() - WALWriter.MAGIC_STRING_V1.getBytes().length - Integer.BYTES; + } + channel.read(metadataSizeBuf, position); + metadataSizeBuf.flip(); + int metadataSize = metadataSizeBuf.getInt(); + endOffset = channel.size() - WALWriter.MAGIC_STRING_BYTES - Integer.BYTES - metadataSize - 1; + } finally { + channel.position(WALWriter.MAGIC_STRING_BYTES); + } + } + private void analyzeFileVersion() throws IOException { if (channel.size() < WALWriter.MAGIC_STRING_BYTES) { version = FileVersion.UNKNOWN; @@ -113,7 +155,7 @@ public void close() throws IOException { @Override public int available() throws IOException { - long size = (channel.size() - channel.position()); + long size = (endOffset - channel.position()); if (!Objects.isNull(dataBuffer)) { size += dataBuffer.limit() - dataBuffer.position(); } @@ -121,6 +163,9 @@ public int available() throws IOException { } private void loadNextSegment() throws IOException { + if (channel.position() >= endOffset) { + throw new IOException("End of file"); + } if (version == FileVersion.V2) { loadNextSegmentV2(); } else if (version == FileVersion.V1) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/WalChecker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/WalChecker.java index cbf2ead7464e..5e53e9f664a6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/WalChecker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/tools/WalChecker.java @@ -22,15 +22,14 @@ import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry; import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryType; import org.apache.iotdb.db.storageengine.dataregion.wal.exception.WALException; +import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALInputStream; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.BufferedInputStream; import java.io.DataInputStream; import java.io.File; -import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.util.ArrayList; @@ -86,8 +85,7 @@ public List doCheck() throws WALException { } private boolean checkFile(File walFile) { - try (DataInputStream logStream = - new DataInputStream(new BufferedInputStream(new FileInputStream(walFile)))) { + try (DataInputStream logStream = new DataInputStream(new WALInputStream(walFile))) { while (logStream.available() > 0) { WALEntry walEntry = WALEntry.deserialize(logStream); if (walEntry.getType() == WALEntryType.WAL_FILE_INFO_END_MARKER) { From 27a15c344541ccc9f0694be43e3e5c3a46580636 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Sun, 26 May 2024 18:03:40 +0800 Subject: [PATCH 17/32] set default to uncompress --- .../src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index f4f841d00b68..773e880e2e0c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -1138,7 +1138,7 @@ public class IoTDBConfig { */ private String RateLimiterType = "FixedIntervalRateLimiter"; - private CompressionType WALCompressionAlgorithm = CompressionType.LZ4; + private CompressionType WALCompressionAlgorithm = CompressionType.UNCOMPRESSED; IoTDBConfig() {} From cd8939fba7c8827812d0636c24a1695aa91c890b Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Mon, 27 May 2024 18:30:11 +0800 Subject: [PATCH 18/32] fix wal ut --- data.zip | Bin 37702 -> 0 bytes .../org/apache/iotdb/db/conf/IoTDBConfig.java | 4 ++-- .../dataregion/wal/io/LogWriter.java | 7 +++++-- .../dataregion/wal/io/WALInputStream.java | 6 +++++- .../dataregion/wal/io/WALMetaData.java | 9 +++++++++ .../dataregion/wal/io/WALReader.java | 8 +++++++- .../wal/recover/WALNodeRecoverTask.java | 1 + .../wal/recover/WALRecoverWriter.java | 4 ++-- .../dataregion/wal/node/WALNodeTest.java | 4 ++++ .../wal/recover/WALRecoverManagerTest.java | 4 ++++ .../wal/recover/WALRecoverWriterTest.java | 1 + 11 files changed, 40 insertions(+), 8 deletions(-) delete mode 100644 data.zip diff --git a/data.zip b/data.zip deleted file mode 100644 index 876ef4eff2ab0bacea5c5683453b9858490044c6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 37702 zcmc$H1z48N@;43ADIJ1zcZhVCv~+iOcb9^6m$WF2gh&fWw@7z4NW=G_uN}wte>m6o z=n7ny%>M3~+1c5dxo4%tLBXJbu0KKnrAi;a`R5-*ARHh)Z98pRT}ulCV?zr|J$+gQ z`CCAsEv6y{Yk*A9emW=ybJ! zK!rYP{#T5sr;fq?0E`QOh4Juzf|2e&!eVRn0xE=s4ZBc(0M5m4)EINs43k z8sBP>4jG_jFFOIt3EPEuONyV!wmABZlGr|)bO?Cu5PCW)r6M^sd^#$&gZh=02G#)A zCwuUfD*=x*?|T1FxkCRJTmb?OjiM73@ zo%UZUuh+L$`+9wmpq^8c0We*U8`NE0GkaS*eVe~R1(^A+K9epTD?KAUHJuIvJvEay zn*p^pvkn_IvoJ7uPGP5@{ zw)iVxUl{h)bggNm7v%zkYY74bbdwCzwy`s|Gq(I&Tv}RsURsYc&(D$T=Ir<(nBY+$ z1>hh-QRK;WJxAsG-S8q1Ltwc z024QU4I);e6?X~}tf8Jqmr6IYdWR06-S+`Q>&AKW35|`V zr5z2xvobI?)BlqRJ-fQPO1su@2ndMkg`=}0K51q@=)5i|?_$ooY6(}2SC9odkohBn z13EK?GmwAKHvRwMlW1NG{BIuTdW-sMMi8~=o`?dNYz`PVXhuHot+u*G`sUjI$p<11 zD}nQ(g9Yoo>vb!P_fd;{^b@btfI8nt_3S6TF2Do!8{G^<4y(%Rb;=4Wskdb!F=a}h z_F*fGh=|Yv@o_X)U>wOx7Ytq z8ltPG=LPbIy;<(&z}o8>{9h$ItloRz0bm_F;E;8LHuO5k@t2T3ZpRStP_4}kEpP_b zSXc?`m_QkK02?x>?-Fx5Dx*U_B*7!ZC?^aIb}gzr-VF$?3cz34iK<7=;L$pbG>711 ztsx3xp`c`~0qUdN#f}79v$G(#=;`w4cz{u&4G<3ZKkOqMfihI1bW6xZMRnQ9$#$mW zHQj;v#_s(Xw7G!@fv&xcjlPB5Uxr7n0mOPQQQ-iDaXoI}B=Z@7jkbXuwYk2XHqEs{ z|0(nY1x$zmeH$8>4jF--ft7|1@Q0p*o{p7)jt%6ip;7=qWk%ZveFHcojAPx%IpH%@ zGfP8&+5mPqLqI&kR+H`{*z2&;FG5S3ajr(H3OH@O)vV!3QD&0ZgKg#mMN{suH!f!jt%Qp zXOqLFFT1BV9I9)&RQVmmWsEL{>jp30oW4(+a(aBVw=dIeVAL8M{)EF1%r66*;Z9GU z8YW>Gc*`Am4hV5h*Hvh~Fkmd3Iny*C&|?OvhHCdlBo(9dna8lNGgV>y6%Qgf-}A8_ zgtKta3PUoi!w z@WKkQep_UzZa-Un{4J${;!5E`gSrnJmGT(wp z=WvfH$cno&2)H%D?1@fWci5yj5cPHU$=GOKu#ko6K#qC!Gho>>oglWSFieFeJQ@u(s>e6>#i_D9VzZ?lScBzEyS>?E6^iaf#q?SS=}XQIghUHnTrQi2|n}k_IAoyp2S53MWSo3*==7a zs+WT;Z8c7HLqYK6hlRa7>1!r4%9Ngovp&8;qqAzaDDbS0o6`ctDRuQvz;`fCFAeqF zSvWJoDVJe`rMC)l-<9Q=B%iQHb{c>9mlKj=7#CrU3Zq}*D?bCg{VX{c#>9=G@qZC!ElBb#67t;`1) zm`s`!TPSQAG^-Fdp|=S*1=x?of(ssJo9V7rqf@ozKO)xBwa%MlC=@sz@FR4KgfVQV zc2!~J5M7M*o)`$+p78J%I^Oeqw6oazf~!E_7`iI=RRjL9K=97vc&J`N>LuY2&st*&~)0S~A+C8KX@kc3+aO^oyH1}>IK5ggVY#6|L zymOG4Ze=6a9n9cO|6*;Z`~LE3l?ldjjl;2*KwC9B9ekh$&Lva7^^3vrHG>s4gfLtD zj(h$4=wHJx*E@ic?jarn!1`Yg>HpqA_-t0c>;PYk|C^t~q9P|tX{BrUH40H@k&<>V%`g%f^HuveE0Wm1^`}EM?W1nW>0=omieLckfd+dNX z1K`j@V`iyq`UUm;kxnb_EP%#7`w?467)^z<4f1qlkud8qDpl|b){59}jMKI!=e`F1dr)Up&@sBfqRM?!mHK9Fcne>1(CITAe3yS3Rj~ zCJDo>RH{@OE2FQv9xA8wV!Ep>jB(`dhr}vK#tpx!a%DHgeI&GCLLEWViv7sam@{=s zV2~}N%w95zteWIGPb0P6fYB7jCQohSZVIyz*5G~)dQbKn2Z;@VGH?>ON^FM$*sDUi z!wsm29DE#~MHMm;RVOt5@G?ydV;ZKgR;I@!qNrB+?%ucWqnn#Hn!f4q6TEzUOAOqN zlu8Vi1|i0^KX-oRlm!}{J$}x$xCcG!!6Lu%{Wt-REui-VClvQyNWzf62ZjoH3?o3v z3JN0Zt1;4nA0fpV2TIwPvRXYwFzUOf(Oin}gN$QG%4~(k7MC98PGrdNDh_^dWsto2 zWj+E{J>(QIH~ihlG*x;@se0VSGK%STW@xFNEbB$ZJGhC!k1kl#WM3zxaJ(z=HTNGc zrA62uuDfdYe2Ol5nY1~>rxPEBG0jtk>1mmAL)z#HT^&2 zuo*t5vA=@&#X;liL;lFNFR|D!?ElKP?;rB7Kzz<;|3_y3h~rBv;l^;#Ytl1(V#*)D zeA-a1CDW=bZ>R!LC?=?H^2s*~#tv{cq_K0dqxmj}bgehP5SgwPAgllw~F?KXUNazO9N|Rw#&ky)G$TF=*pU3nac$`GMTS%Pf!#q zlJ8yy0uxR7Y0zb2>)XDD?C@Q8?^ZsV?sM)cZ5qsl$ajf{n&&UFjuDn?8Q0wA%c@S+ z87GEOUj4wfZ-aKu{~NY_4B&mY27epC`wct42Jk-F-XAT{4KKRSzWPVU>Njfr)e>EU zG4@d$`W!C0VdeV_=F7q1Yv|}2+A?7L;9!0Z9nt>~I%57QbR-t7x=;sLFX{m2Q}T*3oOXop~W64N+nWQrEiOry|*7T79)k{%q!BeNAbG~qLp*%CRBIdnw; zjA#I`c96==@&-^Sb5f$zQZ#R=$U3@uecNYjVT2nuDIo}6S^-Tqz0MYCb^#%ZXd@r) zXjcepWzHeE@-~Oi`LJ+4;S)W?fQXI}MR2c*m#;b)jX0q?pI{sgZpJ6US(;afC?X*F zd`^d;TP<1Ehr6oE&Kz~gE4dxqj?ge5Ag~iZ0;M|pHG|EhuAVgHs=m)D?yO=2ZWnALM6g^lu2aX&lq9GCZbo}OP zJFGo53lsn*;WgIR;00#S=Ic9dJhdN{J~IOh0r^r-RFCPK0PgjI7#>pr`Fwi*lBT$s zUFfrBeOVA+mTA||YvGqCY4%^7r0IV?NnbyQJ)%$S^(p##DBkP}`SDbJy_$c0s%EGA zyHoX#X7TPH0AI`EzjOP@{*pRi>-_4|{lZ*bThg!Y3-%p;2?7AKDFNf&znFnu^V1R8 z-bVX6heZQ;zAt+5i%I7{IVB5PS|Bp81U}djwx6>dN>ZV|-zNQkbnuXW0_CV}_8vsbyP3dg_1pcCFTPKdbile49DXW$6UW;tT@>^yL)xJF=Pp+H`VE`>|FGdn+%uigzU?HGtY)mhEBgsC>y}IAnbxn0bBAEHdb@kKJ>$^@|J@e4;%khKE)kRj_RD$_9` z4$2WgIYEGd-r*rbgzy7J;<@Vup6O)-1la=A>#alp9|Vc)B|vaX1VYwJo&Zh+QUeH+ zhhzu29i-G7hXB1DR0H^g2Y3g|33SH$CC?pcFjP=t9&%}jXizjBVrlSKpatGo>9_Mh z*SwE_fQ^8#yh+kAjeu3WS<-P=fVhBCc+mUc4uI-_kXArXy~7(wwSX|d0=%?m80kR5 zKy(|}=z!gTNxaFmu;PIYyxFmgj!^G9;f3Hln6k@jA*F?zFrK4E=Nd@S#|$%{8x;w< z(ob>UMK+BVkCEon$xGVYe{vBBLO)36v2AfoQ$CC_KREUnFC<8b^9`DW(+arVRI!8! zLI>94AuEa1<|*==xMfGaJ^{LCS3ATMk)SGFt+cyN>J_WKBh7vj^;(z3bTi=$87ZE= z^r-tSZiv{6cjrq{n4E12s^#V{LGA#8`Qi-Sj|gnB7ab?r*_F%L6Q`0MM~5RXEX^(5 zcbthK&05M-=6DvvOm*%cRgb6Ks=rM{u_2-rWmSM2tdUGtW&g|*vX)dVfPt+y#&J?y z`5q~ve^+ZlmW|yr{cy#C7^H2ZCBt-LQ+Tu>_+3iH6c|Uw567CD?v7nbGI=jIOynXm zo`?xIKZ2+c>3x`#hZh=CCX{JdO@;y?!av>i;x^T8vbN^ zt5?aLjte5U9;p-Lm{JmUO;+1+by=8=O{SP%SrBcb$9C2%yRv5%lmu}m$j;Cz^eY7E zOMB?QA&6kkI-uKYC#(K{5YYSXLCS+Rf)ZG8feilEy%>3KYI3&h3zB(5knUR}QVNfx~%$yAf;jp>z9908rWswNC!V z4y(7Wg7}WFa^s`d9&7QI>bPq<(~#t^=!KK-+cy^KGBQ4k5uy<1pvJZ9+@qG0i6b8t z7xbUA+siJwU2+@VtyExC3+I`!+={Dw;xP7G$S3)<<#Kr4Jz7OipYRSX`OmVPbPC?B zGieFZZdWr2-~~(4FY-U}7;WC&bIhf4Z#pSAZ^i2gE2*b2Pp-|;-EYLbW6#*Khq?$B zFUfb>jOAPqMoNbU!#LkaD?WR$ww-P{YQfyS2w$i2R)*FspUdQ~6&;Hk!5m(b2<^+z z103aU?X|%U)*>|I+htP<6v2KaN@yBYeEmb5M3>`|$QkN-nIgMzB!WC^^x6w$yL9UN9NKDREAHx_-MO_VN83m6hlh=-% zaf*_Ib1Oa5WEILZD`$vIlWLS6`0>clGPBt4)ZH$bO{X)Q+||Q$nUMk3M1EwF(y$G| zH6Yn2+0SHRYSQ`Ie7*S91G(LGiqN16yGS zC$vimYW#JtGuK%U!WhrJyq`U{r|GMbMz7C==k71ab){@%WkEZRiay#X%RBACXI5~u z((R!FlDk41Os5#U^qrrJ=;~vYm@pl3zb`W43_fa5H(?vQtdV_MRY|~}kJSI*6g8&+ zF8Bob8Ki;{Tn%s{)qb-cB|=Dx+oldeqEFK8__i~O#`fm2bD!F})bRuvs(ERUTx=W| z6f6rDg4vZ~*%J$su-?0Oc6n&@Rc#fOOH?J}su*SWhDcJ+snVq$njX8O9iCrKt-lwO zt+>NdHyfBHjEqtzkPO3YY(V!I?yzNXg}=**jZ7g?EAQspvI?NE&SfN@PTjG*S~s-VY%)&l)O$)fr8Rt(ED4l~uPzUS7Ui(Qmfca>>zr`joKt^d;B*G)C1-RcIW% zk~I2$`@8Ya)Xex@3k08gm7qA2Yl%!y8%R7>3KW0gnY7&Fq^+Ivp!m!;7SVOZCoze) z!39EdhO`8&F6qKY40K)(t|Gp`)yeOxr|;5*nf1 zgZyCQ*i>Hl?dUuo*W-3>qvU-eZdbAW3x6Y#TJ&9Ze~%;Ig86bHgA0yFA7qoLFDXIk zZNX*krxh0(kIq>nn@g`!jl;^Vxlu(*oG^<-Ar{uK(@dY{csL4x_@1o#!tZUbR?xts z(CsP~7cdn*cgVkOl{X@8Ji?_etXxf2o~=;Z_9|wQhpCc?NF4N~h&c9~eXnH65!$2V z`qD-Lh65>d$@w%w*;!Eh)7(DwSY*yNv~n3sr4o{R7OZu8%MY1`gPeQ#eRf%OL*Qy( zFdzq3t@v~QSB%5INW^bv5TOvg!0 z*de^?Ts?=QNja&b+#}H8=>-_jHS7EBp}znDaY{|-PI?jKQ?@*gWFc$oY*;fPiw@s9vrREh_;aE zwwZhU!T~2SlQ_{aO>LoBB{eF<1@{$KFDfD!{>9Zf?UfWw)Ro5874Y}ovcxK+aR@9B z5HJc55a}-+`fr?YO*(42AAIGppWX4zcg{>huJ{eR(_#8T4stXQL~ps8Fdh-h>xxDO z@s319r|?yZxX=vn#kN%Vy-BSajTMcBx-&+Too6Jbmk)J+y5{BUBd%We9^0Oa^^^Go zTaD#c?n_~5CtagBZ^U9m#*KY=5r-ha8oMmD1*M@L2ZAfZ##iC)Lj{q=5f=22a*tAUsWMZ2xVLL;=b(Ic7<*_ z2hKs1(2ggh+ocoK4rxRU6!jtpwq=wMjhCf+>DI|z=(pZWId-Z~ta^DVmNhG;H{-9b z{kAbWFds+`i3BMJxyzg`rk<3~z}GPH0a_{)GAELWl~{n>;gyF}rKfVp>9i&^IE^6a zOIdPpMp@!Z zSrUlu;I#@hcq@motC>z+tHwk8K28!5AK#q`Y($@NZADBw_Bo###3Q~PNGcx`5Ipu1k(UUvz=%^E%2%Nj&M8N8?nAlpHqy$E^0qP@($7}7!VK<@Iu5rD3} zE2`IIgH8uF0#Wf|DF*EWwgbMeMOqB=4o(`_y#a_0EUZ`)fw%|b9ZWpX8IYV7w-y8r zP!}-F41v}yC19ootYToZ5=~9fWLzx-HlW8|C^N7)5NbdV8{pVLsz8N-5oR!~aCX36 z0a-Rc?tq^Ic@h9Yfk^?8dJ_;}Ks6HDR>_HcNZmTbo2GhIHCa1GzA3-GGH{VB{Ax=* z!_-h}ScP*I)DWQ)n7}s8v)0`V@&3z-qeWn%VUOBfzwp;*8jKam*5zGz2w2Tpu`2nE z1P-KhxYQ%A`|ry;WF5RGw7pxKh*J6M@X1C|bsosn;@f9Ygy=i=lZHwR>DJuF90M++ ztagiLmym)v?H4T4Grm2q-!aK&*td*#ujBshAiGiVG7bFsl%992Li%D{y7!%PJfS?9 z)$`nfXBDvqra~~CNsUj#TPRb7#}zpf^L^}K*F=4fiuMkLAjmq9?$6DKAbT@1W^jwC z({ivH@kDp`Z55+>ypwjV-t>4nMiU5U{TBZCXql2^(dRriVIRE2RRH4&BmX`s5qGuP z81;g`GPeBjsoNXKuJYYs!#tbaVqEt4HW2ze^;r3g(j9?;J((+y{)}WPa3c&pRFh$|Tel|lU8e(u5d1@Ah0Q)_mTV~2Y1IBF9vW?U zIQE*L;h5N`60Xe^fk+XKgC-qK976Te7k41#9+_x3?`8xLb19W(3Ugb#&rdL zTeM0p>PL@VQA|zMUy71Ed#wx(rZU$M*w~O84=LJ}EJS};l6OQ+%zS!wP*$R4$%kH*d3>1&Zl~4jHaFnH_Ct}+SEyfz0)tUtJ73iUNI|p zp@4=y4TVpuVm9wCpVC?r%ztRx(=xdFj*R=r!7ftr^i-^ivO*W_*zgtx-xkk6*Xv%8 z9Cze;5vcZeJ@^Rd3ANL%B#G42O6)b%=PdQNB!|;SI6`!WH*kt9ukyKez#nGP7))n? zLDx3N+$y(A_Nxlb87#M&Z-g*DEK8ak@V_uj9*mLWTjo=S+XxwDbhCwyPkC8{I7o!& z7U7mSX-|>Lhdji8*3FKU744$L zSda=A8xgi-B}lw5n-PpMIF6DB{unIuG59m2Y5FJd{3lUk{ZY}pG>%y>iVe0!xL!a5) zH3f$uN|918{vaino)#HSC6~9E>6za7tUo2`IqP^D`Sy{@ETXKHBUmzm5*P$eI^t8Bvn0!kYq(iYOax6~_K!bx$7?$L*3P z#iJFdo^3jbB(HqX<2W&(UweAjA_aT5Zemd!FGn6P0{#GLSe0rhE*NfZkX8zwc?XTL z%PQ$z>Z7VxuWAX)H&rxR-t@qhX35mf2)gkK&c7&OwHB;Mz#aq>AF*ez+A7bm@droE=%(Cvr>>dBoeby~8ko-RJIh29mZYfV$ zpHbt&z1pzKHb-THg-m9Tp2vy~lOKxTE?g2bb310rO}c4fc9#fHY9Nh|v}5_5#TwC!3D?jf3#Twlp3hqP4;J|9k@~KD{Gu~Iv-q(UU)ie44yk(JhG%hyK`nwwTru|MiQO}y*an%(_fKDh zu&N!}GU%V%W$w&Pq$@17Ba`^el5el|-=m*=Yp)}>>dgP-6;`%sSphQJ9%4n4kMixq z8Ri$h8iJPS74r%qQ$%={Zyp@1;*X7UICxfWJnm{RR?S>kM5j{q6kSapJjRU5nQXPp zeHyR>ApH19SNp-S;)3yR9$lbXmfwZk_uAq|~9C}GlUtP)1 zs)Nu1{hUkQk*tBJ zWu@|@0s$F%Id(45*#lemX^N?|i$(iv}kz;FJ(FVy2wTYoQC$z$TS#w-Zp@^ZLi{*^d&UGfvj)Z1t96 z788Ba(7E@FSjKA^?DDd}EtccbT4cilojq}=>(lqg@^u5n4=vz{wB))QJrkj|EG;`p=dAOXQJt@PZEm%OFX4`m-CJOB3*J@8 z{lMmUuDw8kiUD)5I?W-((QJh$p{MARLm!TjXw^kDQ#FIQ&#q3TB_~B*CQdl@wDbaAK#)Os%JClR{(U3>;Q(H@ydWiHK$z+;|K5y;UpI7se zGe!opSq=b(!}(OHm*f+TD=O5U&zLJ8M{J zX9_mYax1DMj22#ELtBBKUv7KeddR(ZIG@Ws*J6w_M`$$jKfl~+CRve9+5?V)%q?3lMZlT;_$zSnKI#Mdp(GJE+4a{KOC zysB3jHHYB#$3;DGQH{6gthcDFMu8Y(c*WZePgo=^m+N1f8gKro+Xaet&~qpt&aD^}Ca z*j(TCx<2fqSg+^k_!&3-?|~EjPOkq<@@A8)NTKmVp zMY*JZ9cGy}PZCw*b3NtvyPmx^R@dSq0Z^Enqpg`8}y zYqduU#=4Ap<`PEtOD$8$sjW!+bg2#AW_vjnzq6*D=QIP|#R%ib2cTbt^y3=&Qegk1 zc6}=J`aiCVF9r6$q55AgyWg6@e?WgBPxlr7FE+nxu6(Un__pAZ_IIlL^8}xZD8E!G zT-(1N)cAi}JNc<5;YYK9`MXB|yxRJSD}QA*zL+7vH$Rs-{Ilk+N!poPeI@&wrThu$ zkDD6GPe`w|<`c31xI_N<*m%DtW)EoT1$f8ULie9nM*d)LK7stt>*OQI@2UQ#{Nnek z@8@;$J=HIY{HN-U-;@7&o&1LUHx(*>-2Z+Q-Om*&zn$QprT7Pdep-sxQ@jnWvLXj4 zoHSra{92%282^Pu^t5!e^sMwObPRwG9RnQ`HK0ikH64we?R9Sl8i03XX>X(Zsma4{ zd=x-^s+Jazwq68m%Y@*`39(D-YJdxQ06_^vp4x>c7@+>yXDuKh;@fx7ui-uj3L<*} zfZ7Gvm-uf6?iWw$7qxIOdHNjI96BAbiyy zPsOWDAi$6HfFblRXZfl^j0}K7z&9%NCz-@3*-N8NjB42M#zxZc@?wF-T0B8Uiiq8c zgfLdJd9oE55g|psnyp+T&f%ymo5g#4{`$y%>8NcxZM$oN-LYrFdF#I8($sRY@Sx<> z$P_Xt&(_+)LeHi_cc*z{sQ>DlP>_tL8Jb=Wyyyh7ctAjiQ0aX0yr4jRbA*oV6y$i*It~>);$rGbbkLG*WnI6D;;1RV9;`mZ3>=xoc$cnX=A$4Ot6EH8I?Im@X zCf}39E)>*cub&hSPKX55os6$FEQITu3Tml6?I+WHbQbL6VGS$<>gdLOWK$q`b`mT; zO9h`7zOmA!g+;2qC9`B>QP8>EFmynTk{KY)5FoqfA%p_JRansUtqf%~~lAv9(ug%Tk3kEfrWOs5%WfIJaPW9=(QI~H94eux*C z*_?~0^R*oK}PGA<}0G|O(tc?0< zN^f{I>qYG;J^8FRZX@B`_<4PbSwoaQqPJ8jgO_5V3)ta{J4f2SqebE_1$5kohjp6$ zl~ENB%y5iOAX;C^`{ld5mJw&Le5vIM4j0@96t|3T2hUTaZ2>-A-pg#qznJqp*GlHR zbgg1nj!=b3BUwRs%o1q>qR{^p6vw3$g5*A2yD-c9NidDaSq#zDL1Yl!D!zg4&Z^dl z^jkT&M`U6xgeDXye8v+v@&KP?`h4F#OjKz97vxd%E_rQLA zP^^-2QGc&wMUiBz>YeFRssU2eSr+C(>s?-W5*fMW%C0BeL-A^7JfvQ}u_7;1rH*#{ z`VNzV&e=)Pw((?q{dyX@g6Whyvv^n5W!)x+0s`F7gQXHcO{ zHyjDLRJKkzMoOkgkj)WHhr8(ZSy6f<>0bGtK~~Rmv@y{ku-Vb^l9Hl|<|d4uonCrO z9dt6!9YT&mE}PNw8;$%0uR)tbL6`@nJ$ zO@4HctAAQ4@|2fW!Vf=?br?5`{e5WAd*jyP<-#l|Tg@E7aBRV_t7PZY`bMZ|8N2Lg zt@Z?S3W3rmD&AS*Z9?6N3r}$C-(#eb-$~(Bvo@m*NRr$She9h8A-7#iV9>#gG+p5i(4LQ<9=suU$s(d z%sk!075=K(_hHN9_cEpC?lG&jO43o5Y;&7S=AMrPVIoatcLX)H*@51{tM+I2Ts5ST zpt!Sh25RQrbKYO_Pu#W;qDL+01~&^QS8~cbn`BKj+$dJh^}!0p zI%u@+1F9fBBLz&wcqx@%dOam3mxKD=kEhQuTk z;tnyC$Q~|W-hI~pv@_ba_|2=1dEYhRbyYg(hD`z_RF3Do)5Ef`^_fY5#JS|e?n)Xy z&^45&x~ph`7|Z6d1o9@0a#-G!9jj{@(GZ}gF3WY5WiiuuUImhV(_F%y*=ci@+RA&ivP;ujBV17LO8ooCWh{KcKz{^NMNA zw(&ERC|$pl4TlKVoGHcv47Aq8l=D=FGHa45EjYCGop7iWP6NLpB+L7CD4C)EhI)3d_BLm%`k<%<+)R@Wo%-9dvzS1Rg^XN@*4F`ABU%j% zV)~w|967epr^WmQY>C2RwBlnY;*zd*X;=aB5AIEL*pWGn(BQwSpH2)?jtV$fN6NOo zkl2S!=Jp)WUO)3WSV~fJ?7TP|SRynj21pLqkIa&=irXuWN+e|JS;kQ^mgNVy?s&EA z;%a>CXI%3S^E3{$>47Qfts_LwC$Z{wI++O`| zP8EyrrVL3Bd+`WCK+TTeato57H~8#($F41@@&UtE8!G7sgw=a`n$CtC{t}&%8ERf9 z7DsT4p@lv`1G5s|h9OpaPKlX`@Cl2)?WId*NGAT3BnNV=cs-)^2|O@Yh3u(7=Z>?1 z7(t4+`I=xMym3@hOJgrxM}WIXDgx+E{kuask8jDV!^-ws)rReKEo4Pe$y*NciNl-| zn$m(hoN06_uAW!>+AFCZJ8qo0W|BD0^k;ED$xGOA0;dOZYEC$=3be^LTF>P*xDsQ1 zb4RsUMAw2Yyfrl6WibQH1KpG+Dt54g0(Ty|giI>=626ISLkTPK$p7*43LGg1^m*2z z5wnr5lGD6`G9T2;#M8Gc8XLr>FK#`N-sH+TsR9pv!|8g6lT2Edt!Z9O$Fs^?A)@I; z9O(0#OE-T9l9HE3vCkWxO;l82gx7NABL( zU0=(+gEi$bwntKT-!^$I1hfkD_Nk1=R$1UWHHZ>hR{?KO$zBg)vZzbYtN1j93G z>a?zw@+2jfo&rDW&jd=g>Kga1*MQ#SW=PU%$7JT50H1~zMdsXm;oTpgHGwR|i8kP% zxQ7*;XT0B~bE2g;fw&24K!e%fLlW-((2G?Q)o@#R$`Yh2rGN(;XBOAipOgzfI*RbN zwMz6tG+2n1I>feh!Hcx0GVI+0s3+Ucs`E^Da5}^q=|%O>n3<1NAt@45Vm_>ToDMZt~t@-|j*R<==-8}HGKiyP9o;6tcJOd9s99BE7@^mm1<84hVs z;8ObL_M^p8toB+D-8Xx)2}0uDNl=l3_6TFnE|ht(VL>9IwsZ8G^q3o$s9%^{2gux` zx#S~T6|qp3EdEU^K`fDtiXTXx?my4^>JU+d+y1xEO zSfLs6XGXTexa=@@_?=Llj!wBAtZ0QKfUw?w{#=(BO=RC-Na`_$|9(*i_u3Ut>)8g* zZiLAblBLEY-INaHLIXL&=|vK=`vb$zfuUhgY?$0#sZa?kwullK^^r(5Kj^KmRcL#P z#T=IQR?e($9AS6&c9XKZVy05Z>c}$$`chcLL*hUZ4oUS#y-veI&8?1a*gdd$&w@7xaVSI*k# zY)^Fb*FvdGY{3`~J*%{Pf){d0xq2@)t)@v)zuWARPga>P%SGiGc`II=0(GZLBll*y zMmSk9`TAawxkZDDe063-9F=)UnI{?K`BfIq!^+wrd7S3jFcTlyc};O-VG6gTR6&2U zH8rEMYH)oM(O4xm8mXyDi^od><>oJYOQ>{oN|^a{aY+MZDSQ|0&TOdb>6Xifa>qyI zK1fsgSLV9hqIZG6auAgWUmWgx+BP~XmWWUNs3xL!7PK8D>lx-L)yM>s<2jXcxfmgl zf;o9y_-0zXVp(@B?qFN51jJxI;r!tqm(SEYTO1Ue_5wjo57f=Kdo?i(Z!i2>6HV`v zYIr|CKCOHa9E10A%|b0ATxhnZRRH|MqOSFMN3z3*IrctmgI){mdnLgepze2{g~_DT z7|r!?Ea?&;#m7DG=TQKEv>62(I}+Dq3qTcZ=CIZ%7*N4&f3$z}8LJ(V=hiy~0gDHy~fhh>kfCbE&G(4Y64 znF!Zjx^TRD$2i*8=T>)uymF6haWXnR+iklB=xOzWp*POnhbcdZ9Sy^{8o2@%*N3XQ zSxXR7wx!yoxU57gQj=IHPw!yO1-|i#u(Q6>T5t?RWa)0#&R2Q2nK~sldNk=-6()$1}O8@+KYQW$Gd9@Xd$fIw?LZ5cZ7#1lnacJczZ$y@%BC3QxY<{A)3IY z$wwsx9~nLb%e)_On@&`Dht_RyQnf}t8@wqfdyTCFGK%-sen3WVMEkTwlTewYrYc5{ zka;!QikQhR*yZcTL~vXyEePA=SAvO$a4%Y9(W4+qsCEmv~V1!B)OZm1dNW5 zF7zzgc7Qy6-h>Rnjqx(o-nG}^5fm~IyW!|l;#c+dB(3@h)Q(Eaar+WEFZve}<0e@1 zsuM~X9}cg)yd7zLa%v7^2<8+;Cj0?DR3i40=3xk@i@zr&esMWsr$$0ZtR;KDJfT{4 zJF*sOPlM+{^yQjuV2U(R|$p2$jMPlVFP$KgTWgXdR1 z@<2rlJx_=83cZh%?NFUG?hsxjhm87ea$^*>XMc6VtgRV~E&H=nOD)gFwAb^jkF@+h zEW3Ezo1JOpS_oEMvUIme?M+iNWFE5xft}ub+qEsj7ZljqaCG~g`rXDj?j0fYXFGAl zA21w}@5;_4z0@VeQ%&58BZE-2^<>g7h)OM(2)%`qip*lEyej~4JbNxN54B_=)H2;2DHvRSW!kXkKb*Ye*X#IBSWJ`kARi!=2R+3tmQ zbTj8nnEP;e-`4i%Xbqo^0?uH>0gb+JB(JCLDWRf<((X3Ig)qw5Qot1Tj>qd zDIwa6T#!L~1bx9^xkif#!jO}^6>bffTZCR62x(En}^Q8r&%{+C*p4L3p z!CU;hL1fUU7h+l5o(a-4fDGz&7=JlkW(Z-b~4qo9aB zM(&+p7G=v=a&G|Y?}y-cDdwRe-I{4F`#JBRSothNQ| zaOD>0;KSBF)LTCE4N54foAtv`Cs7h42G@VfHzj)uN-sf+5AU*sr-3^FzQJx@u+94l za`8M(^Ws_f`EbeQnyKemcB*Ugc|)*np{KRT)y&St!(`9U$okgZ>6}&CTOX2>xq08$ z$6Y}}+&f2rKdzrWzo@=^U0yG7%vy%Gfaq$gL-B!|Egid@+xdc#)cf3302j9W3KjE8 z=<4u;(G?9sx-ijc86^<*{x&gi))j0r58>(+9HAQ z2Y4|K0`S5&FbE##rx5LrJv$MBFaUAw>klH(ChzsvkMaFauju~M90&ZpQAqzn}`3;L8;(|DHbgcl1;LNdFIAWIlJYxz1etXAnQO&*u9M1Ueu!@FVNKKIC74 z_}Iz@fa90fvV0Zck2r+C!x42OIR3e&<==ym{SFK&pp@yqXUaz~fG%oZuWb1fu1`sV zYbJevVasPd2Lj6SPXC|a`U3HrF1~+<^!saDzC|hkX!vpS9Q@}Jm%jt|{p~5=f>QxB zv%Pt6|Ji7M9Cg1W62+(}{hM;}! z!2b2ZpuYq6?L{09zvEl=4TAgW29rO5`?_$y73%lkq#$pMb3fl?@+WX##riGhl)vNL z!p(vE*oo7{QgFhZt^YtfVcOBpP{;G~|q0QbLw4XaJ0>E7tF8+r|%l8fMKY;_B z`T?5;8{mika?|)M#4mjNTWNmZ%Klqa2md{)?^^KvNrb;vb@VUR;o6Z)!2Fp@-+E-f z)sJ5j$N7%<+V6>fd@JH_+4tlWt@;~r@eDC52vH-~XlD@w z5fl6XLGcsP*;@%>A%cG)q)9FK0a_@g2!frnXP=qbYoB?Y+;a$ANHc4{XU}G49mCO@ z&`~s~1xM*Y8A2R8n}j%gr+LY1LFxFc1Tk;Rcm?1NYT!Ebe(b0=eL6=7PmG_KtE$#f z^b_XAkq;cmc`=n(8Q>?q9+aOST5E=5s!g9ljnp^$FT(17Gplpk8Q3;W7)M`ng<-75 zngQ11v9?ao1hFy$VoA7O7D$^G&#>S7706Aj3l7mTg=UT1{rr5dMsghB`AFgFN8XdK zg0{T(D1G;b5zM_O68z*jw}z$!iPK#clB7Q`3}Qk}qi)1ZdOCYb#n^CbaHe*RT_Q z8(r5uyR+oh1|srKja#~~l_IsZwB3&q1QK1>!67%Ggj={Cvy{+CG6j~w$SuguA;HXx zKoy0G8~o`t;Oa1V(WlBdfj~kFiFb3Yga`B%IkyM@%Hm!MpDL9j!7v~lYcE#B>D`2q z0mNki-+AeXwgM{cbU>pA$Yt&HV{g1j^gD$YCM(f*>EXE6rpe&U;(h=2y@d6exr!#u zF#1&B$%F19*zaQj&+}*GfVlj&*)ybKN7fUHaS8yJhQTVF-gTVg0Bm+Yg=IeyUlj%# z0S9333kY*XI94e0QBGCco1($T4gUoQ^D;dj-eHrJ=gq4=9PGHL9?YE~^PpT7bxxh~ zpj4J7vxJ7sB9V){`106Wyv{Apj1pxC-4>ky4fa|PwkNFFF|=vU!NR8>Xw%q*foY;F zypCnemX~o1PkvyvNVZ<=0gZs!vRgpFZ0QJv*|NM5HJDe%Lp7U00Zh9+3|A>|ck02K z6?T+SPfHqGDe L<=fQ@R diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 773e880e2e0c..20be293b0bd1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -223,7 +223,7 @@ public class IoTDBConfig { private int walBufferQueueCapacity = 500; /** Size threshold of each wal file. Unit: byte */ - private volatile long walFileSizeThresholdInByte = 30 * 1024 * 1024L; + private volatile long walFileSizeThresholdInByte = 30 * 1024L * 1024L; /** Size threshold of each checkpoint file. Unit: byte */ private volatile long checkpointFileSizeThresholdInByte = 3 * 1024 * 1024L; @@ -1884,7 +1884,7 @@ public long getWalFileSizeThresholdInByte() { return walFileSizeThresholdInByte; } - void setWalFileSizeThresholdInByte(long walFileSizeThresholdInByte) { + public void setWalFileSizeThresholdInByte(long walFileSizeThresholdInByte) { this.walFileSizeThresholdInByte = walFileSizeThresholdInByte; } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index b73b3b1e26f0..ebef9f443ec1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -99,12 +99,11 @@ public double write(ByteBuffer buffer) throws IOException { headerBuffer.put( compressed ? compressionAlg.serialize() : CompressionType.UNCOMPRESSED.serialize()); headerBuffer.putInt(bufferSize); + size += 5; if (compressed) { headerBuffer.putInt(uncompressedSize); size += 4; } - size += bufferSize; - size += 5; try { headerBuffer.flip(); logChannel.write(headerBuffer); @@ -150,4 +149,8 @@ public void close() throws IOException { } } } + + public long getOffset() throws IOException { + return logChannel.position(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java index b0872ace299b..f02c48bcda4d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java @@ -215,7 +215,7 @@ private void loadNextSegmentV2() throws IOException { dataBuffer.clear(); unCompressor.uncompress(compressedData, dataBuffer); } else { - dataBuffer = ByteBuffer.allocateDirect(dataBufferSize); + dataBuffer = ByteBuffer.allocate(dataBufferSize); if (channel.read(dataBuffer) != dataBufferSize) { throw new IOException("Unexpected end of file"); } @@ -276,4 +276,8 @@ public void read(ByteBuffer buffer) throws IOException { read(buffer); } } + + public long getFileCurrentPos() throws IOException { + return channel.position(); + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java index 7916af9308e5..c3459dd2f463 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java @@ -49,6 +49,7 @@ public class WALMetaData implements SerializedSize { private final List buffersSize; // memTable ids of this wal file private final Set memTablesId; + private long truncateOffSet = 0; public WALMetaData() { this(ConsensusReqReader.DEFAULT_SEARCH_INDEX, new ArrayList<>(), new HashSet<>()); @@ -164,4 +165,12 @@ private static boolean isValidMagicString(FileChannel channel) throws IOExceptio return magicString.equals(WALWriter.MAGIC_STRING) || magicString.startsWith(WALWriter.MAGIC_STRING_V1); } + + public void setTruncateOffSet(long offset) { + this.truncateOffSet = offset; + } + + public long getTruncateOffSet() { + return truncateOffSet; + } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALReader.java index 475ea2b0b2d8..1310bb36b46f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALReader.java @@ -44,6 +44,7 @@ public class WALReader implements Closeable { private final File logFile; private final boolean fileMayCorrupt; + private final WALInputStream walInputStream; private final DataInputStream logStream; private WALEntry nextEntry; private boolean fileCorrupted = false; @@ -55,7 +56,8 @@ public WALReader(File logFile) throws IOException { public WALReader(File logFile, boolean fileMayCorrupt) throws IOException { this.logFile = logFile; this.fileMayCorrupt = fileMayCorrupt; - this.logStream = new DataInputStream(new WALInputStream(logFile)); + this.walInputStream = new WALInputStream(logFile); + this.logStream = new DataInputStream(walInputStream); } /** Like {@link Iterator#hasNext()}. */ @@ -84,6 +86,10 @@ public boolean hasNext() { return nextEntry != null; } + public long getWALCurrentReadOffset() throws IOException { + return walInputStream.getFileCurrentPos(); + } + /** * Like {@link Iterator#next()}. * diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java index 237b353c9f9a..07a06448359e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALNodeRecoverTask.java @@ -172,6 +172,7 @@ private long[] recoverLastFile() { } } } + metaData.setTruncateOffSet(walReader.getWALCurrentReadOffset()); metaData.add(walEntry.serializedSize(), searchIndex, walEntry.getMemTableId()); } } catch (Exception e) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriter.java index 7015d45c58b0..5db0c17c9870 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriter.java @@ -41,14 +41,14 @@ public WALRecoverWriter(File logFile) { public void recover(WALMetaData metaData) throws IOException { // locate broken data - int truncateSize; + long truncateSize; if (logFile.length() < MAGIC_STRING_BYTES) { // file without magic string truncateSize = 0; } else { if (readTailMagic().equals(MAGIC_STRING)) { // complete file return; } else { // file with broken magic string - truncateSize = metaData.getBuffersSize().stream().mapToInt(Integer::intValue).sum(); + truncateSize = metaData.getTruncateOffSet(); } } // truncate broken data diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNodeTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNodeTest.java index 61f5e4150e53..a3c2e71e4d2a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNodeTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/node/WALNodeTest.java @@ -80,9 +80,12 @@ public class WALNodeTest { private WALMode prevMode; private String prevConsensus; private WALNode walNode; + private long originWALThreshold = + IoTDBDescriptor.getInstance().getConfig().getWalFileSizeThresholdInByte(); @Before public void setUp() throws Exception { + IoTDBDescriptor.getInstance().getConfig().setWalFileSizeThresholdInByte(2 * 1024 * 1024); EnvironmentUtils.cleanDir(logDirectory); prevMode = config.getWalMode(); prevConsensus = config.getDataRegionConsensusProtocolClass(); @@ -93,6 +96,7 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { + IoTDBDescriptor.getInstance().getConfig().setWalFileSizeThresholdInByte(originWALThreshold); walNode.close(); config.setWalMode(prevMode); config.setDataRegionConsensusProtocolClass(prevConsensus); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java index 7193d7722a53..0ba1ebf25717 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverManagerTest.java @@ -109,9 +109,12 @@ public class WALRecoverManagerTest { private CheckpointManager checkpointManager; private TsFileResource tsFileWithWALResource; private TsFileResource tsFileWithoutWALResource; + private long originWALThreshold = + IoTDBDescriptor.getInstance().getConfig().getWalFileSizeThresholdInByte(); @Before public void setUp() throws Exception { + IoTDBDescriptor.getInstance().getConfig().setWalFileSizeThresholdInByte(1 * 1024 * 1024); EnvironmentUtils.cleanDir(new File(FILE_WITH_WAL_NAME).getParent()); EnvironmentUtils.envSetUp(); prevMode = config.getWalMode(); @@ -122,6 +125,7 @@ public void setUp() throws Exception { @After public void tearDown() throws Exception { + IoTDBDescriptor.getInstance().getConfig().setWalFileSizeThresholdInByte(originWALThreshold); if (tsFileWithWALResource != null) { tsFileWithWALResource.close(); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriterTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriterTest.java index f492f015ec22..97b00f379f23 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriterTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/recover/WALRecoverWriterTest.java @@ -169,6 +169,7 @@ public void testFileWithBrokenMagicString() throws IOException, IllegalPathExcep walMetaData.add(size, 1, walEntry.getMemTableId()); try (WALWriter walWriter = new WALWriter(logFile)) { walWriter.write(buffer.getBuffer(), walMetaData); + walMetaData.setTruncateOffSet(walWriter.getOffset()); } long len = logFile.length(); try (FileChannel channel = FileChannel.open(logFile.toPath(), StandardOpenOption.APPEND)) { From 610a2e2fbe563fad928945f34ede11c9b174972b Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Mon, 27 May 2024 18:32:43 +0800 Subject: [PATCH 19/32] optimize calculating of wal size --- .../iotdb/db/storageengine/dataregion/wal/io/LogWriter.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index ebef9f443ec1..1ceee59eaff3 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -99,11 +99,10 @@ public double write(ByteBuffer buffer) throws IOException { headerBuffer.put( compressed ? compressionAlg.serialize() : CompressionType.UNCOMPRESSED.serialize()); headerBuffer.putInt(bufferSize); - size += 5; if (compressed) { headerBuffer.putInt(uncompressedSize); - size += 4; } + size += headerBuffer.position(); try { headerBuffer.flip(); logChannel.write(headerBuffer); From 8929d419f75fbd0d3bf74c8fbb397ed14e68bdf0 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Mon, 27 May 2024 18:39:02 +0800 Subject: [PATCH 20/32] close wal file when the origin size of wal buffer is larger than threshold --- .../db/storageengine/dataregion/wal/buffer/WALBuffer.java | 3 ++- .../db/storageengine/dataregion/wal/io/LogWriter.java | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java index 5096caf6783e..ecda70843ca9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java @@ -540,7 +540,8 @@ public void run() { boolean forceSuccess = false; // try to roll log writer if (info.rollWALFileWriterListener != null - || (forceFlag && currentWALFileWriter.size() >= config.getWalFileSizeThresholdInByte())) { + || (forceFlag + && currentWALFileWriter.oiginalSize() >= config.getWalFileSizeThresholdInByte())) { try { rollLogWriter(searchIndex, currentWALFileWriter.getWalFileStatus()); forceSuccess = true; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index 1ceee59eaff3..e7cdcf9b69e0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -45,7 +45,8 @@ public abstract class LogWriter implements ILogWriter { protected final File logFile; protected final FileOutputStream logStream; protected final FileChannel logChannel; - protected long size; + protected long size = 0; + protected long originalSize = 0; private final ByteBuffer headerBuffer = ByteBuffer.allocate(Integer.BYTES * 2 + 1); private static final CompressionType compressionAlg = IoTDBDescriptor.getInstance().getConfig().getWALCompressionAlgorithm(); @@ -77,6 +78,7 @@ public double write(ByteBuffer buffer) throws IOException { if (bufferSize == 0) { return 1.0; } + originalSize += bufferSize; buffer.flip(); boolean compressed = false; int uncompressedSize = bufferSize; @@ -130,6 +132,10 @@ public long size() { return size; } + public long oiginalSize() { + return originalSize; + } + @Override public File getLogFile() { return logFile; From b2d667ffa4e339f7500aa6ab1c840ebbf3c8808c Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Thu, 30 May 2024 18:43:11 +0800 Subject: [PATCH 21/32] add the size of magic string --- .../iotdb/db/storageengine/dataregion/wal/io/LogWriter.java | 1 + .../db/storageengine/dataregion/wal/io/WALInputStream.java | 3 ++- .../iotdb/db/storageengine/dataregion/wal/io/WALWriter.java | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index e7cdcf9b69e0..3f9419e6ed67 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -61,6 +61,7 @@ protected LogWriter(File logFile) throws IOException { this.logChannel = this.logStream.getChannel(); if (!logFile.exists() || logFile.length() == 0) { this.logChannel.write(ByteBuffer.wrap(WALWriter.MAGIC_STRING.getBytes())); + size += logChannel.position(); } if (compressionAlg != CompressionType.UNCOMPRESSED) { compressedByteBuffer = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java index f02c48bcda4d..73dbb2948984 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java @@ -260,9 +260,10 @@ public void skipToGivenPosition(long pos) throws IOException { } dataBuffer = ByteBuffer.allocate(currSegmentSize); channel.read(dataBuffer); + dataBuffer.flip(); dataBuffer.position((int) posRemain); } else { - dataBuffer.clear(); + dataBuffer = null; channel.position(pos); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java index 469dcaa8c698..c5be73892e78 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java @@ -75,6 +75,7 @@ private void endFile() throws IOException { buffer.putInt(metaDataSize); // add magic string buffer.put(MAGIC_STRING.getBytes()); + size += buffer.position(); writeMetadata(buffer); } From 3297ffb9f0f38571bfaff963702714fae4017081 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Thu, 30 May 2024 23:47:54 +0800 Subject: [PATCH 22/32] may be fix the bug --- .../dataregion/wal/buffer/WALBuffer.java | 4 +- .../dataregion/wal/io/LogWriter.java | 2 +- .../wal/compression/WALCompressionTest.java | 54 +++++++++++++++++++ 3 files changed, 57 insertions(+), 3 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/compression/WALCompressionTest.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java index ecda70843ca9..76ee06c2e1c6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java @@ -541,7 +541,7 @@ public void run() { // try to roll log writer if (info.rollWALFileWriterListener != null || (forceFlag - && currentWALFileWriter.oiginalSize() >= config.getWalFileSizeThresholdInByte())) { + && currentWALFileWriter.originalSize() >= config.getWalFileSizeThresholdInByte())) { try { rollLogWriter(searchIndex, currentWALFileWriter.getWalFileStatus()); forceSuccess = true; @@ -583,7 +583,7 @@ public void run() { position += fsyncListener.getWalEntryHandler().getSize(); } } - lastFsyncPosition = currentWALFileWriter.size(); + lastFsyncPosition = currentWALFileWriter.originalSize(); } WRITING_METRICS.recordWALBufferEntriesCount(info.fsyncListeners.size()); WRITING_METRICS.recordSyncWALBufferCost(System.nanoTime() - startTime, forceFlag); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index 3f9419e6ed67..76dd47e0ebfe 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -133,7 +133,7 @@ public long size() { return size; } - public long oiginalSize() { + public long originalSize() { return originalSize; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/compression/WALCompressionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/compression/WALCompressionTest.java new file mode 100644 index 000000000000..79c82a92320f --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/compression/WALCompressionTest.java @@ -0,0 +1,54 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.storageengine.dataregion.wal.compression; + +import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileStatus; +import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileUtils; +import org.apache.iotdb.db.utils.constant.TestConstant; + +import org.junit.After; +import org.junit.Before; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; + +public class WALCompressionTest { + private final File walFile = + new File( + TestConstant.BASE_OUTPUT_PATH.concat( + WALFileUtils.getLogFileName(0, 0, WALFileStatus.CONTAINS_SEARCH_INDEX))); + + @Before + public void setUp() throws IOException { + if (walFile.getParentFile().exists()) { + Files.delete(walFile.getParentFile().toPath()); + } + Files.createDirectory(walFile.getParentFile().toPath()); + } + + @After + public void tearDown() throws IOException { + if (walFile.getParentFile().exists()) { + Files.delete(walFile.getParentFile().toPath()); + } + } + + public void testSkipToGivenPosition() {} +} From b3653476b50cb293a57acd878ec2aa74b200d94b Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Fri, 31 May 2024 21:04:18 +0800 Subject: [PATCH 23/32] fix with comment --- .../main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 2 +- .../java/org/apache/iotdb/db/conf/IoTDBDescriptor.java | 7 +++---- .../db/storageengine/dataregion/wal/io/WALInputStream.java | 6 +++++- .../src/assembly/resources/conf/iotdb-system.properties | 6 +++--- 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 20be293b0bd1..1c324babca73 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -223,7 +223,7 @@ public class IoTDBConfig { private int walBufferQueueCapacity = 500; /** Size threshold of each wal file. Unit: byte */ - private volatile long walFileSizeThresholdInByte = 30 * 1024L * 1024L; + private volatile long walFileSizeThresholdInByte = 30 * 1024 * 1024L; /** Size threshold of each checkpoint file. Unit: byte */ private volatile long checkpointFileSizeThresholdInByte = 3 * 1024 * 1024L; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index a3c4fec1e19d..9d2be034e99f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -417,11 +417,10 @@ public void loadProperties(Properties properties) throws BadNodeUrlException, IO properties.getProperty( "io_task_queue_size_for_flushing", Integer.toString(conf.getIoTaskQueueSizeForFlushing())))); - + boolean enableWALCompression = + Boolean.parseBoolean(properties.getProperty("enable_wal_compression", "false")); conf.setWALCompressionAlgorithm( - CompressionType.valueOf( - properties.getProperty( - "wal_compression_algorithm", conf.getWALCompressionAlgorithm().toString()))); + enableWALCompression ? CompressionType.LZ4 : CompressionType.UNCOMPRESSED); conf.setCompactionScheduleIntervalInMs( Long.parseLong( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java index 73dbb2948984..8aeeb6d83db9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java @@ -250,10 +250,14 @@ public void skipToGivenPosition(long pos) throws IOException { buffer.clear(); channel.read(buffer); buffer.flip(); - buffer.get(); + CompressionType type = CompressionType.deserialize(buffer.get()); currSegmentSize = buffer.getInt(); if (posRemain >= currSegmentSize) { posRemain -= currSegmentSize; + channel.position( + channel.position() + + currSegmentSize + + (type == CompressionType.UNCOMPRESSED ? 0 : Integer.BYTES)); } else { break; } diff --git a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties index 7c64edaa900b..7dbf5f5b7165 100644 --- a/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties +++ b/iotdb-core/node-commons/src/assembly/resources/conf/iotdb-system.properties @@ -1431,9 +1431,9 @@ data_replication_factor=1 # Datatype: long # iot_consensus_cache_window_time_in_ms=-1 -# WAL compression algorithm -# options: UNCOMPRESSED, SNAPPY, LZ4, GZIP, ZSTD -# wal_compress_algorithm=UNCOMPRESSED +# Enable Write Ahead Log compression. +# Option: true, false +# enable_wal_compression=false #################### ### IoTConsensus Configuration From 443d1a095cbb6e0658f8fe755aee7a85deb6c785 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Sun, 9 Jun 2024 16:00:51 +0800 Subject: [PATCH 24/32] edit with review --- .../AbstractNodeAllocationStrategy.java | 3 -- .../dataregion/wal/buffer/WALBuffer.java | 6 ++-- .../dataregion/wal/io/LogWriter.java | 4 +-- .../dataregion/wal/io/WALInputStream.java | 35 +++++++++++++++---- .../dataregion/wal/io/WALMetaData.java | 2 +- 5 files changed, 34 insertions(+), 16 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/allocation/AbstractNodeAllocationStrategy.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/allocation/AbstractNodeAllocationStrategy.java index 9c8ebd35aac5..e1d6a832baab 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/allocation/AbstractNodeAllocationStrategy.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/allocation/AbstractNodeAllocationStrategy.java @@ -73,9 +73,6 @@ protected IWALNode createWALNode(String identifier) { protected IWALNode createWALNode(String identifier, String folder) { try { return new WALNode(identifier, folder); - } catch (FileNotFoundException e) { - logger.error("Fail to create wal node", e); - return WALFakeNode.getFailureInstance(e); } catch (IOException e) { logger.error("Meet exception when creating wal node", e); return WALFakeNode.getFailureInstance(e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java index 76ee06c2e1c6..627771bde385 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java @@ -520,9 +520,9 @@ public void run() { forceFlag, syncingBuffer.position(), syncingBuffer.capacity(), usedRatio * 100); // flush buffer to os - double compressionRate = 1.0; + double compressionRatio = 1.0; try { - compressionRate = currentWALFileWriter.write(syncingBuffer, info.metaData); + compressionRatio = currentWALFileWriter.write(syncingBuffer, info.metaData); } catch (Throwable e) { logger.error( "Fail to sync wal node-{}'s buffer, change system mode to error.", identifier, e); @@ -535,7 +535,7 @@ public void run() { memTableIdsOfWal .computeIfAbsent(currentWALFileVersion, memTableIds -> new HashSet<>()) .addAll(info.metaData.getMemTablesId()); - checkpointManager.updateCostOfActiveMemTables(info.memTableId2WalDiskUsage, compressionRate); + checkpointManager.updateCostOfActiveMemTables(info.memTableId2WalDiskUsage, compressionRatio); boolean forceSuccess = false; // try to roll log writer diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index 76dd47e0ebfe..0fdf831ad543 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -53,7 +53,7 @@ public abstract class LogWriter implements ILogWriter { private final ICompressor compressor = ICompressor.getCompressor(compressionAlg); private final ByteBuffer compressedByteBuffer; // Minimum size to compress, default is 32 KB - private static final long MIN_COMPRESS_SIZE = 0; + private static long minCompressionSize = 32 * 1024L; protected LogWriter(File logFile) throws IOException { this.logFile = logFile; @@ -85,7 +85,7 @@ public double write(ByteBuffer buffer) throws IOException { int uncompressedSize = bufferSize; if (compressionAlg != CompressionType.UNCOMPRESSED /* Do not compress buffer that is less than min size */ - && bufferSize > MIN_COMPRESS_SIZE) { + && bufferSize > minCompressionSize) { compressedByteBuffer.clear(); compressor.compress(buffer, compressedByteBuffer); buffer = compressedByteBuffer; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java index 8aeeb6d83db9..d540bba35884 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java @@ -37,6 +37,7 @@ public class WALInputStream extends InputStream implements AutoCloseable { private final ByteBuffer headerBuffer = ByteBuffer.allocate(Integer.BYTES + 1); private final ByteBuffer compressedHeader = ByteBuffer.allocate(Integer.BYTES); private ByteBuffer dataBuffer = null; + private ByteBuffer compressedBuffer = null; private long fileSize; File logFile; private long endOffset = -1; @@ -205,17 +206,37 @@ private void loadNextSegmentV2() throws IOException { } compressedHeader.flip(); int uncompressedSize = compressedHeader.getInt(); - dataBuffer = ByteBuffer.allocateDirect(uncompressedSize); - ByteBuffer compressedData = ByteBuffer.allocateDirect(dataBufferSize); - if (channel.read(compressedData) != dataBufferSize) { + + if (Objects.isNull(dataBuffer) + || dataBuffer.capacity() < uncompressedSize + || dataBuffer.capacity() > uncompressedSize * 2) { + dataBuffer = ByteBuffer.allocateDirect(uncompressedSize); + } + dataBuffer.clear(); + + if (Objects.isNull(compressedBuffer) + || compressedBuffer.capacity() < dataBufferSize + || compressedBuffer.capacity() > dataBufferSize * 2) { + compressedBuffer = ByteBuffer.allocateDirect(dataBufferSize); + } else { + compressedBuffer.clear(); + } + + if (channel.read(compressedBuffer) != dataBufferSize) { throw new IOException("Unexpected end of file"); } - compressedData.flip(); + compressedBuffer.flip(); + IUnCompressor unCompressor = IUnCompressor.getUnCompressor(compressionType); - dataBuffer.clear(); - unCompressor.uncompress(compressedData, dataBuffer); + unCompressor.uncompress(compressedBuffer, dataBuffer); } else { - dataBuffer = ByteBuffer.allocate(dataBufferSize); + // The case of UNCOMPRESSION + if (Objects.isNull(dataBuffer) + || dataBuffer.capacity() < dataBufferSize + || dataBuffer.capacity() > dataBufferSize * 2) { + dataBuffer = ByteBuffer.allocateDirect(dataBufferSize); + } + dataBuffer.clear(); if (channel.read(dataBuffer) != dataBufferSize) { throw new IOException("Unexpected end of file"); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java index c3459dd2f463..8824bff3a26a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java @@ -163,7 +163,7 @@ private static boolean isValidMagicString(FileChannel channel) throws IOExceptio magicStringBytes.flip(); String magicString = new String(magicStringBytes.array()); return magicString.equals(WALWriter.MAGIC_STRING) - || magicString.startsWith(WALWriter.MAGIC_STRING_V1); + || magicString.contains(WALWriter.MAGIC_STRING_V1); } public void setTruncateOffSet(long offset) { From 130930e97085c0afbd5d09b9671e48fef5674873 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Mon, 10 Jun 2024 21:43:17 +0800 Subject: [PATCH 25/32] fix test --- .../dataregion/wal/io/WALInputStream.java | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java index d540bba35884..e8cc31ae7274 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java @@ -214,14 +214,7 @@ private void loadNextSegmentV2() throws IOException { } dataBuffer.clear(); - if (Objects.isNull(compressedBuffer) - || compressedBuffer.capacity() < dataBufferSize - || compressedBuffer.capacity() > dataBufferSize * 2) { - compressedBuffer = ByteBuffer.allocateDirect(dataBufferSize); - } else { - compressedBuffer.clear(); - } - + compressedBuffer = ByteBuffer.allocateDirect(dataBufferSize); if (channel.read(compressedBuffer) != dataBufferSize) { throw new IOException("Unexpected end of file"); } @@ -231,12 +224,9 @@ private void loadNextSegmentV2() throws IOException { unCompressor.uncompress(compressedBuffer, dataBuffer); } else { // The case of UNCOMPRESSION - if (Objects.isNull(dataBuffer) - || dataBuffer.capacity() < dataBufferSize - || dataBuffer.capacity() > dataBufferSize * 2) { - dataBuffer = ByteBuffer.allocateDirect(dataBufferSize); - } - dataBuffer.clear(); + // TODO: reuse the buffer, and make the buffer read the correct bytes + dataBuffer = ByteBuffer.allocateDirect(dataBufferSize); + if (channel.read(dataBuffer) != dataBufferSize) { throw new IOException("Unexpected end of file"); } From ad021a7eb5e7eb50b0d7978db3f73dcc4375f149 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Tue, 11 Jun 2024 16:50:32 +0800 Subject: [PATCH 26/32] add test for wal compression --- .../AbstractNodeAllocationStrategy.java | 6 +- .../dataregion/wal/io/LogWriter.java | 2 +- .../dataregion/wal/io/WALInputStream.java | 74 +++-- .../dataregion/wal/WALTestUtils.java | 90 ++++++ .../wal/compression/WALCompressionTest.java | 298 +++++++++++++++++- 5 files changed, 433 insertions(+), 37 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/WALTestUtils.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/allocation/AbstractNodeAllocationStrategy.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/allocation/AbstractNodeAllocationStrategy.java index e1d6a832baab..e98086e51462 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/allocation/AbstractNodeAllocationStrategy.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/allocation/AbstractNodeAllocationStrategy.java @@ -32,7 +32,6 @@ import org.slf4j.LoggerFactory; import java.io.File; -import java.io.FileNotFoundException; import java.io.IOException; import java.util.Arrays; @@ -83,11 +82,8 @@ protected IWALNode createWALNode( String identifier, String folder, long startFileVersion, long startSearchIndex) { try { return new WALNode(identifier, folder, startFileVersion, startSearchIndex); - } catch (FileNotFoundException e) { - logger.error("Fail to create wal node", e); - return WALFakeNode.getFailureInstance(e); } catch (IOException e) { - logger.error("Meet exception when creating wal node", e); + logger.error("Fail to create wal node", e); return WALFakeNode.getFailureInstance(e); } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index 0fdf831ad543..b3147da85a40 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -48,7 +48,7 @@ public abstract class LogWriter implements ILogWriter { protected long size = 0; protected long originalSize = 0; private final ByteBuffer headerBuffer = ByteBuffer.allocate(Integer.BYTES * 2 + 1); - private static final CompressionType compressionAlg = + private final CompressionType compressionAlg = IoTDBDescriptor.getInstance().getConfig().getWALCompressionAlgorithm(); private final ICompressor compressor = ICompressor.getCompressor(compressionAlg); private final ByteBuffer compressedByteBuffer; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java index e8cc31ae7274..13bde5d197d1 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java @@ -34,7 +34,7 @@ public class WALInputStream extends InputStream implements AutoCloseable { private static final Logger logger = LoggerFactory.getLogger(WALInputStream.class); private final FileChannel channel; - private final ByteBuffer headerBuffer = ByteBuffer.allocate(Integer.BYTES + 1); + private final ByteBuffer segmentHeaderBuffer = ByteBuffer.allocate(Integer.BYTES + Byte.BYTES); private final ByteBuffer compressedHeader = ByteBuffer.allocate(Integer.BYTES); private ByteBuffer dataBuffer = null; private ByteBuffer compressedBuffer = null; @@ -191,14 +191,14 @@ private void loadNextSegmentV1() throws IOException { } private void loadNextSegmentV2() throws IOException { - headerBuffer.clear(); - if (channel.read(headerBuffer) != Integer.BYTES + 1) { + segmentHeaderBuffer.clear(); + if (channel.read(segmentHeaderBuffer) != Integer.BYTES + 1) { throw new IOException("Unexpected end of file"); } // compressionType originalSize compressedSize - headerBuffer.flip(); - CompressionType compressionType = CompressionType.deserialize(headerBuffer.get()); - int dataBufferSize = headerBuffer.getInt(); + segmentHeaderBuffer.flip(); + CompressionType compressionType = CompressionType.deserialize(segmentHeaderBuffer.get()); + int dataBufferSize = segmentHeaderBuffer.getInt(); if (compressionType != CompressionType.UNCOMPRESSED) { compressedHeader.clear(); if (channel.read(compressedHeader) != Integer.BYTES) { @@ -254,28 +254,32 @@ private void tryLoadSegment() throws IOException { public void skipToGivenPosition(long pos) throws IOException { if (version == FileVersion.V2) { channel.position(WALWriter.MAGIC_STRING_BYTES); - ByteBuffer buffer = ByteBuffer.allocate(Byte.BYTES + Integer.BYTES); long posRemain = pos; - int currSegmentSize = 0; - while (posRemain > 0) { - buffer.clear(); - channel.read(buffer); - buffer.flip(); - CompressionType type = CompressionType.deserialize(buffer.get()); - currSegmentSize = buffer.getInt(); - if (posRemain >= currSegmentSize) { - posRemain -= currSegmentSize; - channel.position( - channel.position() - + currSegmentSize - + (type == CompressionType.UNCOMPRESSED ? 0 : Integer.BYTES)); + SegmentInfo segmentInfo = null; + do { + segmentInfo = getNextSegmentInfo(); + if (posRemain >= segmentInfo.uncompressedSize) { + posRemain -= segmentInfo.uncompressedSize; + channel.position(channel.position() + segmentInfo.dataInDiskSize); } else { break; } + } while (posRemain >= 0); + + if (segmentInfo.compressionType != CompressionType.UNCOMPRESSED) { + compressedBuffer = ByteBuffer.allocate(segmentInfo.dataInDiskSize); + channel.read(compressedBuffer); + compressedBuffer.flip(); + IUnCompressor unCompressor = IUnCompressor.getUnCompressor(segmentInfo.compressionType); + dataBuffer = ByteBuffer.allocate(segmentInfo.uncompressedSize); + unCompressor.uncompress(compressedBuffer, dataBuffer); + } else { + long p = channel.position(); + dataBuffer = ByteBuffer.allocate(segmentInfo.dataInDiskSize); + channel.read(dataBuffer); + dataBuffer.flip(); } - dataBuffer = ByteBuffer.allocate(currSegmentSize); - channel.read(dataBuffer); - dataBuffer.flip(); + dataBuffer.position((int) posRemain); } else { dataBuffer = null; @@ -296,4 +300,28 @@ public void read(ByteBuffer buffer) throws IOException { public long getFileCurrentPos() throws IOException { return channel.position(); } + + private SegmentInfo getNextSegmentInfo() throws IOException { + segmentHeaderBuffer.clear(); + channel.read(segmentHeaderBuffer); + segmentHeaderBuffer.flip(); + SegmentInfo info = new SegmentInfo(); + info.compressionType = CompressionType.deserialize(segmentHeaderBuffer.get()); + info.dataInDiskSize = segmentHeaderBuffer.getInt(); + if (info.compressionType != CompressionType.UNCOMPRESSED) { + compressedHeader.clear(); + channel.read(compressedHeader); + compressedHeader.flip(); + info.uncompressedSize = compressedHeader.getInt(); + } else { + info.uncompressedSize = info.dataInDiskSize; + } + return info; + } + + private class SegmentInfo { + public CompressionType compressionType; + public int dataInDiskSize; + public int uncompressedSize; + } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/WALTestUtils.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/WALTestUtils.java new file mode 100644 index 000000000000..8f671eb3f03d --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/WALTestUtils.java @@ -0,0 +1,90 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.db.storageengine.dataregion.wal; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.PartialPath; +import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.PlanNodeId; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; + +import org.apache.tsfile.common.conf.TSFileConfig; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.write.schema.MeasurementSchema; + +import java.lang.reflect.Field; + +public class WALTestUtils { + public static void setMinCompressionSize(long size) + throws NoSuchFieldException, ClassNotFoundException, IllegalAccessException { + Class logWriterClass = + Class.forName("org.apache.iotdb.db.storageengine.dataregion.wal.io.LogWriter"); + Field minCompressionSizeField = logWriterClass.getDeclaredField("minCompressionSize"); + minCompressionSizeField.setAccessible(true); + minCompressionSizeField.setLong(null, size); + } + + public static long getMinCompressionSize() + throws ClassNotFoundException, NoSuchFieldException, IllegalAccessException { + Class logWriterClass = + Class.forName("org.apache.iotdb.db.storageengine.dataregion.wal.io.LogWriter"); + Field minCompressionSizeField = logWriterClass.getDeclaredField("minCompressionSize"); + minCompressionSizeField.setAccessible(true); + return minCompressionSizeField.getLong(null); + } + + public static InsertRowNode getInsertRowNode(String devicePath, long time) + throws IllegalPathException, QueryProcessException { + TSDataType[] dataTypes = + new TSDataType[] { + TSDataType.DOUBLE, + TSDataType.FLOAT, + TSDataType.INT64, + TSDataType.INT32, + TSDataType.BOOLEAN, + TSDataType.TEXT + }; + + Object[] columns = new Object[6]; + columns[0] = 1.0d; + columns[1] = 2f; + columns[2] = 10000L; + columns[3] = 100; + columns[4] = false; + columns[5] = new Binary("hh" + 0, TSFileConfig.STRING_CHARSET); + + InsertRowNode node = + new InsertRowNode( + new PlanNodeId(""), + new PartialPath(devicePath), + false, + new String[] {"s1", "s2", "s3", "s4", "s5", "s6"}, + dataTypes, + time, + columns, + false); + MeasurementSchema[] schemas = new MeasurementSchema[6]; + for (int i = 0; i < 6; i++) { + schemas[i] = new MeasurementSchema("s" + (i + 1), dataTypes[i]); + } + node.setMeasurementSchemas(schemas); + return node; + } +} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/compression/WALCompressionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/compression/WALCompressionTest.java index 79c82a92320f..8ea4f0e63d3a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/compression/WALCompressionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/compression/WALCompressionTest.java @@ -18,16 +18,45 @@ */ package org.apache.iotdb.db.storageengine.dataregion.wal.compression; +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.db.conf.IoTDBDescriptor; +import org.apache.iotdb.db.exception.query.QueryProcessException; +import org.apache.iotdb.db.queryengine.plan.planner.plan.node.write.InsertRowNode; +import org.apache.iotdb.db.storageengine.dataregion.wal.WALTestUtils; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALBuffer; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntry; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALEntryType; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALInfoEntry; +import org.apache.iotdb.db.storageengine.dataregion.wal.buffer.WALSignalEntry; +import org.apache.iotdb.db.storageengine.dataregion.wal.io.LogWriter; +import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALByteBufReader; +import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALInputStream; +import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALReader; +import org.apache.iotdb.db.storageengine.dataregion.wal.io.WALWriter; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileStatus; import org.apache.iotdb.db.storageengine.dataregion.wal.utils.WALFileUtils; import org.apache.iotdb.db.utils.constant.TestConstant; +import org.apache.commons.io.FileUtils; +import org.apache.tsfile.compress.ICompressor; +import org.apache.tsfile.compress.IUnCompressor; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.PublicBAOS; import org.junit.After; +import org.junit.Assert; import org.junit.Before; +import org.junit.Test; +import java.io.BufferedInputStream; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.File; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.file.Files; +import java.util.ArrayList; +import java.util.List; public class WALCompressionTest { private final File walFile = @@ -35,20 +64,273 @@ public class WALCompressionTest { TestConstant.BASE_OUTPUT_PATH.concat( WALFileUtils.getLogFileName(0, 0, WALFileStatus.CONTAINS_SEARCH_INDEX))); + private final String compressionDir = + TestConstant.OUTPUT_DATA_DIR.concat(File.separator + "wal-compression"); + + private final String devicePath = "root.sg.d1"; + long originalMinCompressionSize; + CompressionType originCompressionType = + IoTDBDescriptor.getInstance().getConfig().getWALCompressionAlgorithm(); + @Before - public void setUp() throws IOException { - if (walFile.getParentFile().exists()) { - Files.delete(walFile.getParentFile().toPath()); + public void setUp() + throws IOException, NoSuchFieldException, ClassNotFoundException, IllegalAccessException { + if (walFile.exists()) { + FileUtils.delete(walFile); + } + originalMinCompressionSize = WALTestUtils.getMinCompressionSize(); + if (new File(compressionDir).exists()) { + FileUtils.forceDelete(new File(compressionDir)); } - Files.createDirectory(walFile.getParentFile().toPath()); } @After - public void tearDown() throws IOException { - if (walFile.getParentFile().exists()) { - Files.delete(walFile.getParentFile().toPath()); + public void tearDown() + throws IOException, NoSuchFieldException, ClassNotFoundException, IllegalAccessException { + if (walFile.exists()) { + FileUtils.delete(walFile); + } + if (new File(compressionDir).exists()) { + FileUtils.forceDelete(new File(compressionDir)); + } + WALTestUtils.setMinCompressionSize(originalMinCompressionSize); + IoTDBDescriptor.getInstance().getConfig().setWALCompressionAlgorithm(originCompressionType); + } + + @Test + public void testSkipToGivenPositionWithCompression() + throws NoSuchFieldException, + ClassNotFoundException, + IllegalAccessException, + QueryProcessException, + IllegalPathException, + IOException { + WALTestUtils.setMinCompressionSize(0L); + IoTDBDescriptor.getInstance().getConfig().setWALCompressionAlgorithm(CompressionType.LZ4); + testSkipToGivenPosition(); + } + + @Test + public void testSkipToGivenPositionWithoutCompression() + throws NoSuchFieldException, + ClassNotFoundException, + IllegalAccessException, + QueryProcessException, + IllegalPathException, + IOException { + WALTestUtils.setMinCompressionSize(1024 * 32); + testSkipToGivenPosition(); + } + + public void testSkipToGivenPosition() + throws QueryProcessException, IllegalPathException, IOException { + LogWriter writer = new WALWriter(walFile); + ByteBuffer buffer = ByteBuffer.allocate(1024 * 4); + List> positionAndEntryPairList = new ArrayList<>(); + int memTableId = 0; + long fileOffset = 0; + for (int i = 0; i < 100; ) { + InsertRowNode insertRowNode = WALTestUtils.getInsertRowNode(devicePath + memTableId, i); + long serializedSize = insertRowNode.serializedSize(); + if (buffer.remaining() >= serializedSize) { + int pos = buffer.position(); + insertRowNode.serialize(buffer); + positionAndEntryPairList.add(new Pair<>(fileOffset, insertRowNode)); + fileOffset += buffer.position() - pos; + i++; + } else { + writer.write(buffer); + buffer.clear(); + } + } + if (buffer.position() != 0) { + writer.write(buffer); + } + writer.close(); + try (WALInputStream stream = new WALInputStream(walFile)) { + for (int i = 0; i < 100; ++i) { + Pair positionAndNodePair = positionAndEntryPairList.get(i); + stream.skipToGivenPosition(positionAndNodePair.left); + /* + Add the allocated buffer size by 2, because the actual serialized size + of InsertRowNode is larger than the estimated value got by serializedSize. + I don't know if this is a bug or not. + */ + ByteBuffer nodeBuffer1 = + ByteBuffer.allocate(positionAndNodePair.right.serializedSize() + 2); + stream.read(nodeBuffer1); + ByteBuffer nodeBuffer2 = + ByteBuffer.allocate(positionAndNodePair.right.serializedSize() + 2); + positionAndNodePair.right.serialize(nodeBuffer2); + nodeBuffer2.flip(); + Assert.assertArrayEquals(nodeBuffer1.array(), nodeBuffer2.array()); + } + } + } + + @Test + public void testUncompressedWALStructure() + throws QueryProcessException, IllegalPathException, IOException { + PublicBAOS baos = new PublicBAOS(); + DataOutputStream dataOutputStream = new DataOutputStream(baos); + List insertRowNodes = new ArrayList<>(); + for (int i = 0; i < 100; ++i) { + InsertRowNode node = WALTestUtils.getInsertRowNode(devicePath, i); + insertRowNodes.add(node); + node.serialize(dataOutputStream); + } + dataOutputStream.close(); + ByteBuffer buf = ByteBuffer.wrap(baos.toByteArray()); + // Do not compress it + IoTDBDescriptor.getInstance() + .getConfig() + .setWALCompressionAlgorithm(CompressionType.UNCOMPRESSED); + try (WALWriter writer = new WALWriter(walFile)) { + buf.position(buf.limit()); + writer.write(buf); + } + + try (DataInputStream dataInputStream = + new DataInputStream(new BufferedInputStream(Files.newInputStream(walFile.toPath())))) { + byte[] magicStringBytes = new byte[WALWriter.MAGIC_STRING_BYTES]; + // head magic string + dataInputStream.readFully(magicStringBytes); + Assert.assertEquals(WALWriter.MAGIC_STRING, new String(magicStringBytes)); + Assert.assertEquals( + CompressionType.UNCOMPRESSED, CompressionType.deserialize(dataInputStream.readByte())); + Assert.assertEquals(buf.array().length, dataInputStream.readInt()); + ByteBuffer dataBuf = ByteBuffer.allocate(buf.array().length); + dataInputStream.readFully(dataBuf.array()); + Assert.assertArrayEquals(buf.array(), dataBuf.array()); + Assert.assertEquals( + new WALSignalEntry(WALEntryType.WAL_FILE_INFO_END_MARKER), + WALEntry.deserialize(dataInputStream)); + ByteBuffer metadataBuf = ByteBuffer.allocate(12 + Integer.BYTES); + dataInputStream.readFully(metadataBuf.array()); + // Tail magic string + dataInputStream.readFully(magicStringBytes); + Assert.assertEquals(WALWriter.MAGIC_STRING, new String(magicStringBytes)); } } - public void testSkipToGivenPosition() {} + @Test + public void testCompressedWALStructure() + throws IOException, + QueryProcessException, + IllegalPathException, + NoSuchFieldException, + ClassNotFoundException, + IllegalAccessException { + PublicBAOS baos = new PublicBAOS(); + DataOutputStream dataOutputStream = new DataOutputStream(baos); + List insertRowNodes = new ArrayList<>(); + for (int i = 0; i < 100; ++i) { + InsertRowNode node = WALTestUtils.getInsertRowNode(devicePath, i); + insertRowNodes.add(node); + node.serialize(dataOutputStream); + } + dataOutputStream.close(); + ByteBuffer buf = ByteBuffer.wrap(baos.toByteArray()); + // Do not compress it + IoTDBDescriptor.getInstance().getConfig().setWALCompressionAlgorithm(CompressionType.LZ4); + WALTestUtils.setMinCompressionSize(0); + try (WALWriter writer = new WALWriter(walFile)) { + buf.position(buf.limit()); + writer.write(buf); + } + ICompressor compressor = ICompressor.getCompressor(CompressionType.LZ4); + byte[] compressed = compressor.compress(buf.array()); + + try (DataInputStream dataInputStream = + new DataInputStream(new BufferedInputStream(Files.newInputStream(walFile.toPath())))) { + byte[] magicStringBytes = new byte[WALWriter.MAGIC_STRING_BYTES]; + // head magic string + dataInputStream.readFully(magicStringBytes); + Assert.assertEquals(WALWriter.MAGIC_STRING, new String(magicStringBytes)); + Assert.assertEquals( + CompressionType.LZ4, CompressionType.deserialize(dataInputStream.readByte())); + Assert.assertEquals(compressed.length, dataInputStream.readInt()); + Assert.assertEquals(buf.array().length, dataInputStream.readInt()); + ByteBuffer dataBuf = ByteBuffer.allocate(compressed.length); + dataInputStream.readFully(dataBuf.array()); + Assert.assertArrayEquals(compressed, dataBuf.array()); + IUnCompressor unCompressor = IUnCompressor.getUnCompressor(CompressionType.LZ4); + Assert.assertArrayEquals(unCompressor.uncompress(compressed), buf.array()); + Assert.assertEquals( + new WALSignalEntry(WALEntryType.WAL_FILE_INFO_END_MARKER), + WALEntry.deserialize(dataInputStream)); + ByteBuffer metadataBuf = ByteBuffer.allocate(12 + Integer.BYTES); + dataInputStream.readFully(metadataBuf.array()); + // Tail magic string + dataInputStream.readFully(magicStringBytes); + Assert.assertEquals(WALWriter.MAGIC_STRING, new String(magicStringBytes)); + } + } + + @Test + public void testWALReaderWithoutCompression() + throws QueryProcessException, IllegalPathException, IOException, InterruptedException { + IoTDBDescriptor.getInstance() + .getConfig() + .setWALCompressionAlgorithm(CompressionType.UNCOMPRESSED); + testWALReader(); + } + + @Test + public void testWALReaderWithCompression() + throws QueryProcessException, + IllegalPathException, + IOException, + InterruptedException, + NoSuchFieldException, + ClassNotFoundException, + IllegalAccessException { + IoTDBDescriptor.getInstance().getConfig().setWALCompressionAlgorithm(CompressionType.LZ4); + WALTestUtils.setMinCompressionSize(0); + testWALReader(); + } + + public void testWALReader() + throws IOException, QueryProcessException, IllegalPathException, InterruptedException { + File dir = new File(compressionDir); + if (!dir.exists()) { + dir.mkdirs(); + } + WALBuffer walBuffer = new WALBuffer("", compressionDir); + List entryList = new ArrayList<>(); + for (int i = 0; i < 100; ++i) { + InsertRowNode node = WALTestUtils.getInsertRowNode(devicePath, i); + WALEntry entry = new WALInfoEntry(0, node); + walBuffer.write(entry); + entryList.add(entry); + } + long sleepTime = 0; + while (!walBuffer.isAllWALEntriesConsumed()) { + Thread.sleep(100); + sleepTime += 100; + if (sleepTime > 10_000) { + Assert.fail("It has been too long for all entries to be consumed"); + } + } + walBuffer.close(); + + File[] walFiles = WALFileUtils.listAllWALFiles(new File(compressionDir)); + Assert.assertNotNull(walFiles); + Assert.assertEquals(1, walFiles.length); + List readWALEntryList = new ArrayList<>(); + try (WALReader reader = new WALReader(walFiles[0])) { + while (reader.hasNext()) { + readWALEntryList.add(reader.next()); + } + } + Assert.assertEquals(entryList, readWALEntryList); + + try (WALByteBufReader reader = new WALByteBufReader(walFiles[0])) { + for (int i = 0; i < 100; ++i) { + Assert.assertTrue(reader.hasNext()); + ByteBuffer buffer = reader.next(); + Assert.assertEquals(entryList.get(i).serializedSize(), buffer.array().length); + } + } + } } From 964bf9b657b006faf3ee344403edb2a00ca57672 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Tue, 11 Jun 2024 17:02:09 +0800 Subject: [PATCH 27/32] add hot reload --- .../apache/iotdb/db/conf/IoTDBDescriptor.java | 4 + .../dataregion/wal/io/LogWriter.java | 28 +++++-- .../wal/compression/WALCompressionTest.java | 73 +++++++++++++++++++ 3 files changed, 98 insertions(+), 7 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java index 9d2be034e99f..dcf8792a145e 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBDescriptor.java @@ -1798,6 +1798,10 @@ public void loadHotModifiedProps(Properties properties) throws QueryProcessExcep properties.getProperty( "merge_threshold_of_explain_analyze", String.valueOf(conf.getMergeThresholdOfExplainAnalyze())))); + boolean enableWALCompression = + Boolean.parseBoolean(properties.getProperty("enable_wal_compression", "false")); + conf.setWALCompressionAlgorithm( + enableWALCompression ? CompressionType.LZ4 : CompressionType.UNCOMPRESSED); // update Consensus config reloadConsensusProps(properties); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index b3147da85a40..98a42b75280d 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -34,6 +34,7 @@ import java.nio.ByteBuffer; import java.nio.channels.ClosedChannelException; import java.nio.channels.FileChannel; +import java.util.Objects; /** * LogWriter writes the binary logs into a file, including writing {@link WALEntry} into .wal file @@ -48,10 +49,10 @@ public abstract class LogWriter implements ILogWriter { protected long size = 0; protected long originalSize = 0; private final ByteBuffer headerBuffer = ByteBuffer.allocate(Integer.BYTES * 2 + 1); - private final CompressionType compressionAlg = - IoTDBDescriptor.getInstance().getConfig().getWALCompressionAlgorithm(); - private final ICompressor compressor = ICompressor.getCompressor(compressionAlg); - private final ByteBuffer compressedByteBuffer; + private ICompressor compressor = + ICompressor.getCompressor( + IoTDBDescriptor.getInstance().getConfig().getWALCompressionAlgorithm()); + private ByteBuffer compressedByteBuffer; // Minimum size to compress, default is 32 KB private static long minCompressionSize = 32 * 1024L; @@ -63,7 +64,9 @@ protected LogWriter(File logFile) throws IOException { this.logChannel.write(ByteBuffer.wrap(WALWriter.MAGIC_STRING.getBytes())); size += logChannel.position(); } - if (compressionAlg != CompressionType.UNCOMPRESSED) { + if (IoTDBDescriptor.getInstance().getConfig().getWALCompressionAlgorithm() + != CompressionType.UNCOMPRESSED) { + // TODO: Use a dynamic strategy to enlarge the buffer size compressedByteBuffer = ByteBuffer.allocate( compressor.getMaxBytesForCompression( @@ -75,6 +78,8 @@ protected LogWriter(File logFile) throws IOException { @Override public double write(ByteBuffer buffer) throws IOException { + CompressionType compressionType = + IoTDBDescriptor.getInstance().getConfig().getWALCompressionAlgorithm(); int bufferSize = buffer.position(); if (bufferSize == 0) { return 1.0; @@ -83,10 +88,19 @@ public double write(ByteBuffer buffer) throws IOException { buffer.flip(); boolean compressed = false; int uncompressedSize = bufferSize; - if (compressionAlg != CompressionType.UNCOMPRESSED + if (compressionType != CompressionType.UNCOMPRESSED /* Do not compress buffer that is less than min size */ && bufferSize > minCompressionSize) { + if (Objects.isNull(compressedByteBuffer)) { + compressedByteBuffer = + ByteBuffer.allocate( + compressor.getMaxBytesForCompression( + IoTDBDescriptor.getInstance().getConfig().getWalBufferSize())); + } compressedByteBuffer.clear(); + if (compressor.getType() != compressionType) { + compressor = ICompressor.getCompressor(compressionType); + } compressor.compress(buffer, compressedByteBuffer); buffer = compressedByteBuffer; bufferSize = buffer.position(); @@ -100,7 +114,7 @@ public double write(ByteBuffer buffer) throws IOException { */ headerBuffer.clear(); headerBuffer.put( - compressed ? compressionAlg.serialize() : CompressionType.UNCOMPRESSED.serialize()); + compressed ? compressionType.serialize() : CompressionType.UNCOMPRESSED.serialize()); headerBuffer.putInt(bufferSize); if (compressed) { headerBuffer.putInt(uncompressedSize); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/compression/WALCompressionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/compression/WALCompressionTest.java index 8ea4f0e63d3a..7ab33343b5d5 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/compression/WALCompressionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/compression/WALCompressionTest.java @@ -333,4 +333,77 @@ public void testWALReader() } } } + + @Test + public void testHotLoad() + throws IOException, + QueryProcessException, + IllegalPathException, + InterruptedException, + NoSuchFieldException, + ClassNotFoundException, + IllegalAccessException { + File dir = new File(compressionDir); + if (!dir.exists()) { + dir.mkdirs(); + } + WALTestUtils.setMinCompressionSize(0); + IoTDBDescriptor.getInstance() + .getConfig() + .setWALCompressionAlgorithm(CompressionType.UNCOMPRESSED); + // Do not compress wal for these entries + WALBuffer walBuffer = new WALBuffer("", compressionDir); + List entryList = new ArrayList<>(); + for (int i = 0; i < 50; ++i) { + InsertRowNode node = WALTestUtils.getInsertRowNode(devicePath, i); + WALEntry entry = new WALInfoEntry(0, node); + walBuffer.write(entry); + entryList.add(entry); + } + long sleepTime = 0; + while (!walBuffer.isAllWALEntriesConsumed()) { + Thread.sleep(100); + sleepTime += 100; + if (sleepTime > 10_000) { + Assert.fail("It has been too long for all entries to be consumed"); + } + } + + // compress wal for these entries + IoTDBDescriptor.getInstance().getConfig().setWALCompressionAlgorithm(CompressionType.LZ4); + for (int i = 50; i < 100; ++i) { + InsertRowNode node = WALTestUtils.getInsertRowNode(devicePath, i); + WALEntry entry = new WALInfoEntry(0, node); + walBuffer.write(entry); + entryList.add(entry); + } + sleepTime = 0; + while (!walBuffer.isAllWALEntriesConsumed()) { + Thread.sleep(100); + sleepTime += 100; + if (sleepTime > 10_000) { + Assert.fail("It has been too long for all entries to be consumed"); + } + } + walBuffer.close(); + + File[] walFiles = WALFileUtils.listAllWALFiles(new File(compressionDir)); + Assert.assertNotNull(walFiles); + Assert.assertEquals(1, walFiles.length); + List readWALEntryList = new ArrayList<>(); + try (WALReader reader = new WALReader(walFiles[0])) { + while (reader.hasNext()) { + readWALEntryList.add(reader.next()); + } + } + Assert.assertEquals(entryList, readWALEntryList); + + try (WALByteBufReader reader = new WALByteBufReader(walFiles[0])) { + for (int i = 0; i < 100; ++i) { + Assert.assertTrue(reader.hasNext()); + ByteBuffer buffer = reader.next(); + Assert.assertEquals(entryList.get(i).serializedSize(), buffer.array().length); + } + } + } } From 0f17197a6ff343bf2a017c0a56c2ea012e7a18da Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Tue, 11 Jun 2024 17:46:38 +0800 Subject: [PATCH 28/32] clean the code to make it more readable --- .../dataregion/wal/io/WALInputStream.java | 57 +++++++++---------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java index 13bde5d197d1..dfb5fafb5a0f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java @@ -40,6 +40,12 @@ public class WALInputStream extends InputStream implements AutoCloseable { private ByteBuffer compressedBuffer = null; private long fileSize; File logFile; + /* + The WAL file consist of following parts: + [MagicString] [Segment 1] [Segment 2] ... [Segment N] [Metadata] [MagicString] + The endOffset indicates the maximum offset that a segment can reach. + Aka, the last byte of the last segment. + */ private long endOffset = -1; enum FileVersion { @@ -60,6 +66,7 @@ public WALInputStream(File logFile) throws IOException { private void getEndOffset() throws IOException { if (channel.size() < WALWriter.MAGIC_STRING_BYTES + Integer.BYTES) { + // An broken file endOffset = channel.size(); return; } @@ -67,16 +74,20 @@ private void getEndOffset() throws IOException { long position; try { if (version == FileVersion.V2) { + // New Version ByteBuffer magicStringBuffer = ByteBuffer.allocate(WALWriter.MAGIC_STRING_BYTES); channel.read(magicStringBuffer, channel.size() - WALWriter.MAGIC_STRING_BYTES); magicStringBuffer.flip(); if (!new String(magicStringBuffer.array()).equals(WALWriter.MAGIC_STRING)) { - // this is a broken wal file + // This is a broken wal file endOffset = channel.size(); return; + } else { + // This is a normal wal file + position = channel.size() - WALWriter.MAGIC_STRING_BYTES - Integer.BYTES; } - position = channel.size() - WALWriter.MAGIC_STRING_BYTES - Integer.BYTES; } else { + // Old version ByteBuffer magicStringBuffer = ByteBuffer.allocate(WALWriter.MAGIC_STRING_V1.getBytes().length); channel.read( @@ -86,9 +97,11 @@ private void getEndOffset() throws IOException { // this is a broken wal file endOffset = channel.size(); return; + } else { + position = channel.size() - WALWriter.MAGIC_STRING_V1.getBytes().length - Integer.BYTES; } - position = channel.size() - WALWriter.MAGIC_STRING_V1.getBytes().length - Integer.BYTES; } + // Read the meta data size channel.read(metadataSizeBuf, position); metadataSizeBuf.flip(); int metadataSize = metadataSizeBuf.getInt(); @@ -191,43 +204,30 @@ private void loadNextSegmentV1() throws IOException { } private void loadNextSegmentV2() throws IOException { - segmentHeaderBuffer.clear(); - if (channel.read(segmentHeaderBuffer) != Integer.BYTES + 1) { - throw new IOException("Unexpected end of file"); - } - // compressionType originalSize compressedSize - segmentHeaderBuffer.flip(); - CompressionType compressionType = CompressionType.deserialize(segmentHeaderBuffer.get()); - int dataBufferSize = segmentHeaderBuffer.getInt(); - if (compressionType != CompressionType.UNCOMPRESSED) { - compressedHeader.clear(); - if (channel.read(compressedHeader) != Integer.BYTES) { - throw new IOException("Unexpected end of file"); - } - compressedHeader.flip(); - int uncompressedSize = compressedHeader.getInt(); - + SegmentInfo segmentInfo = getNextSegmentInfo(); + if (segmentInfo.compressionType != CompressionType.UNCOMPRESSED) { + // A compressed segment if (Objects.isNull(dataBuffer) - || dataBuffer.capacity() < uncompressedSize - || dataBuffer.capacity() > uncompressedSize * 2) { - dataBuffer = ByteBuffer.allocateDirect(uncompressedSize); + || dataBuffer.capacity() < segmentInfo.uncompressedSize + || dataBuffer.capacity() > segmentInfo.uncompressedSize * 2) { + dataBuffer = ByteBuffer.allocateDirect(segmentInfo.uncompressedSize); } dataBuffer.clear(); - compressedBuffer = ByteBuffer.allocateDirect(dataBufferSize); - if (channel.read(compressedBuffer) != dataBufferSize) { + compressedBuffer = ByteBuffer.allocateDirect(segmentInfo.dataInDiskSize); + if (channel.read(compressedBuffer) != segmentInfo.dataInDiskSize) { throw new IOException("Unexpected end of file"); } compressedBuffer.flip(); - IUnCompressor unCompressor = IUnCompressor.getUnCompressor(compressionType); + IUnCompressor unCompressor = IUnCompressor.getUnCompressor(segmentInfo.compressionType); unCompressor.uncompress(compressedBuffer, dataBuffer); } else { - // The case of UNCOMPRESSION + // An uncompressed segment // TODO: reuse the buffer, and make the buffer read the correct bytes - dataBuffer = ByteBuffer.allocateDirect(dataBufferSize); + dataBuffer = ByteBuffer.allocateDirect(segmentInfo.dataInDiskSize); - if (channel.read(dataBuffer) != dataBufferSize) { + if (channel.read(dataBuffer) != segmentInfo.dataInDiskSize) { throw new IOException("Unexpected end of file"); } } @@ -274,7 +274,6 @@ public void skipToGivenPosition(long pos) throws IOException { dataBuffer = ByteBuffer.allocate(segmentInfo.uncompressedSize); unCompressor.uncompress(compressedBuffer, dataBuffer); } else { - long p = channel.position(); dataBuffer = ByteBuffer.allocate(segmentInfo.dataInDiskSize); channel.read(dataBuffer); dataBuffer.flip(); From e48b7dba43234be52b695b960e136a0cb3f2109a Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Wed, 12 Jun 2024 13:38:37 +0800 Subject: [PATCH 29/32] reuse the byte buffer if possible --- .../dataregion/wal/io/WALInputStream.java | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java index dfb5fafb5a0f..780be52671c4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java @@ -214,7 +214,14 @@ private void loadNextSegmentV2() throws IOException { } dataBuffer.clear(); - compressedBuffer = ByteBuffer.allocateDirect(segmentInfo.dataInDiskSize); + if (Objects.isNull(compressedBuffer) + || compressedBuffer.capacity() < segmentInfo.dataInDiskSize + || compressedBuffer.capacity() > segmentInfo.dataInDiskSize * 2) { + compressedBuffer = ByteBuffer.allocateDirect(segmentInfo.dataInDiskSize); + } + compressedBuffer.clear(); + // limit the buffer to prevent it from reading too much byte than expected + compressedBuffer.limit(segmentInfo.dataInDiskSize); if (channel.read(compressedBuffer) != segmentInfo.dataInDiskSize) { throw new IOException("Unexpected end of file"); } @@ -224,8 +231,14 @@ private void loadNextSegmentV2() throws IOException { unCompressor.uncompress(compressedBuffer, dataBuffer); } else { // An uncompressed segment - // TODO: reuse the buffer, and make the buffer read the correct bytes - dataBuffer = ByteBuffer.allocateDirect(segmentInfo.dataInDiskSize); + if (Objects.isNull(dataBuffer) + || dataBuffer.capacity() < segmentInfo.dataInDiskSize + || dataBuffer.capacity() > segmentInfo.dataInDiskSize * 2) { + dataBuffer = ByteBuffer.allocateDirect(segmentInfo.dataInDiskSize); + } + dataBuffer.clear(); + // limit the buffer to prevent it from reading too much byte than expected + dataBuffer.limit(segmentInfo.dataInDiskSize); if (channel.read(dataBuffer) != segmentInfo.dataInDiskSize) { throw new IOException("Unexpected end of file"); From 76cc13905964001e206da9f52acc6beb2894929d Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Mon, 17 Jun 2024 14:45:58 +0800 Subject: [PATCH 30/32] Indicate the encoding of String --- .../dataregion/wal/buffer/WALBuffer.java | 1 + .../dataregion/wal/io/LogWriter.java | 4 +++- .../dataregion/wal/io/WALByteBufReader.java | 1 + .../dataregion/wal/io/WALInputStream.java | 19 +++++++++++++------ .../dataregion/wal/io/WALMetaData.java | 3 ++- .../dataregion/wal/io/WALWriter.java | 5 +++-- 6 files changed, 23 insertions(+), 10 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java index 627771bde385..20970aea34b8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/buffer/WALBuffer.java @@ -540,6 +540,7 @@ public void run() { boolean forceSuccess = false; // try to roll log writer if (info.rollWALFileWriterListener != null + // TODO: Control the wal file by the number of WALEntry || (forceFlag && currentWALFileWriter.originalSize() >= config.getWalFileSizeThresholdInByte())) { try { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java index 98a42b75280d..29335efde5cc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/LogWriter.java @@ -34,6 +34,7 @@ import java.nio.ByteBuffer; import java.nio.channels.ClosedChannelException; import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; import java.util.Objects; /** @@ -61,7 +62,8 @@ protected LogWriter(File logFile) throws IOException { this.logStream = new FileOutputStream(logFile, true); this.logChannel = this.logStream.getChannel(); if (!logFile.exists() || logFile.length() == 0) { - this.logChannel.write(ByteBuffer.wrap(WALWriter.MAGIC_STRING.getBytes())); + this.logChannel.write( + ByteBuffer.wrap(WALWriter.MAGIC_STRING.getBytes(StandardCharsets.UTF_8))); size += logChannel.position(); } if (IoTDBDescriptor.getInstance().getConfig().getWALCompressionAlgorithm() diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java index 3e0909d6c52e..882b5ea468c9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALByteBufReader.java @@ -66,6 +66,7 @@ public boolean hasNext() { */ public ByteBuffer next() throws IOException { int size = sizeIterator.next(); + // TODO: Reuse this buffer ByteBuffer buffer = ByteBuffer.allocate(size); /* Notice, we don't need to flip the buffer after calling diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java index 780be52671c4..7cdcb63095f8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java @@ -28,6 +28,7 @@ import java.io.InputStream; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; import java.util.Objects; public class WALInputStream extends InputStream implements AutoCloseable { @@ -78,7 +79,8 @@ private void getEndOffset() throws IOException { ByteBuffer magicStringBuffer = ByteBuffer.allocate(WALWriter.MAGIC_STRING_BYTES); channel.read(magicStringBuffer, channel.size() - WALWriter.MAGIC_STRING_BYTES); magicStringBuffer.flip(); - if (!new String(magicStringBuffer.array()).equals(WALWriter.MAGIC_STRING)) { + if (!new String(magicStringBuffer.array(), StandardCharsets.UTF_8) + .equals(WALWriter.MAGIC_STRING)) { // This is a broken wal file endOffset = channel.size(); return; @@ -89,16 +91,21 @@ private void getEndOffset() throws IOException { } else { // Old version ByteBuffer magicStringBuffer = - ByteBuffer.allocate(WALWriter.MAGIC_STRING_V1.getBytes().length); + ByteBuffer.allocate(WALWriter.MAGIC_STRING_V1.getBytes(StandardCharsets.UTF_8).length); channel.read( - magicStringBuffer, channel.size() - WALWriter.MAGIC_STRING_V1.getBytes().length); + magicStringBuffer, + channel.size() - WALWriter.MAGIC_STRING_V1.getBytes(StandardCharsets.UTF_8).length); magicStringBuffer.flip(); - if (!new String(magicStringBuffer.array()).equals(WALWriter.MAGIC_STRING_V1)) { + if (!new String(magicStringBuffer.array(), StandardCharsets.UTF_8) + .equals(WALWriter.MAGIC_STRING_V1)) { // this is a broken wal file endOffset = channel.size(); return; } else { - position = channel.size() - WALWriter.MAGIC_STRING_V1.getBytes().length - Integer.BYTES; + position = + channel.size() + - WALWriter.MAGIC_STRING_V1.getBytes(StandardCharsets.UTF_8).length + - Integer.BYTES; } } // Read the meta data size @@ -127,7 +134,7 @@ private boolean isCurrentVersion() throws IOException { channel.position(0); ByteBuffer buffer = ByteBuffer.allocate(WALWriter.MAGIC_STRING_BYTES); channel.read(buffer); - return new String(buffer.array()).equals(WALWriter.MAGIC_STRING); + return new String(buffer.array(), StandardCharsets.UTF_8).equals(WALWriter.MAGIC_STRING); } @Override diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java index 8824bff3a26a..7fa634ffbdcb 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALMetaData.java @@ -29,6 +29,7 @@ import java.io.IOException; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashSet; import java.util.List; @@ -161,7 +162,7 @@ private static boolean isValidMagicString(FileChannel channel) throws IOExceptio ByteBuffer magicStringBytes = ByteBuffer.allocate(WALWriter.MAGIC_STRING_BYTES); channel.read(magicStringBytes, channel.size() - WALWriter.MAGIC_STRING_BYTES); magicStringBytes.flip(); - String magicString = new String(magicStringBytes.array()); + String magicString = new String(magicStringBytes.array(), StandardCharsets.UTF_8); return magicString.equals(WALWriter.MAGIC_STRING) || magicString.contains(WALWriter.MAGIC_STRING_V1); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java index c5be73892e78..7017b4be6cb0 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALWriter.java @@ -30,13 +30,14 @@ import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; /** WALWriter writes the binary {@link WALEntry} into .wal file. */ public class WALWriter extends LogWriter { private static final Logger logger = LoggerFactory.getLogger(WALWriter.class); public static final String MAGIC_STRING_V1 = "WAL"; public static final String MAGIC_STRING = "V2-WAL"; - public static final int MAGIC_STRING_BYTES = MAGIC_STRING.getBytes().length; + public static final int MAGIC_STRING_BYTES = MAGIC_STRING.getBytes(StandardCharsets.UTF_8).length; private WALFileStatus walFileStatus = WALFileStatus.CONTAINS_NONE_SEARCH_INDEX; // wal files' metadata @@ -74,7 +75,7 @@ private void endFile() throws IOException { metaData.serialize(logFile, buffer); buffer.putInt(metaDataSize); // add magic string - buffer.put(MAGIC_STRING.getBytes()); + buffer.put(MAGIC_STRING.getBytes(StandardCharsets.UTF_8)); size += buffer.position(); writeMetadata(buffer); } From 581481245f19bdaa65bd49457341992da5a21d20 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Mon, 17 Jun 2024 15:04:57 +0800 Subject: [PATCH 31/32] Edit according to comment --- .../dataregion/wal/io/WALInputStream.java | 21 ++++++++++++++++++- .../wal/utils/WALEntryPosition.java | 4 +++- .../wal/compression/WALCompressionTest.java | 4 ++-- 3 files changed, 25 insertions(+), 4 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java index 7cdcb63095f8..844c06436b76 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/io/WALInputStream.java @@ -18,6 +18,8 @@ */ package org.apache.iotdb.db.storageengine.dataregion.wal.io; +import org.apache.iotdb.db.utils.MmapUtil; + import org.apache.tsfile.compress.IUnCompressor; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.slf4j.Logger; @@ -27,6 +29,7 @@ import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; +import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; import java.nio.charset.StandardCharsets; import java.util.Objects; @@ -217,6 +220,9 @@ private void loadNextSegmentV2() throws IOException { if (Objects.isNull(dataBuffer) || dataBuffer.capacity() < segmentInfo.uncompressedSize || dataBuffer.capacity() > segmentInfo.uncompressedSize * 2) { + if (!Objects.isNull(dataBuffer)) { + MmapUtil.clean((MappedByteBuffer) dataBuffer); + } dataBuffer = ByteBuffer.allocateDirect(segmentInfo.uncompressedSize); } dataBuffer.clear(); @@ -224,6 +230,9 @@ private void loadNextSegmentV2() throws IOException { if (Objects.isNull(compressedBuffer) || compressedBuffer.capacity() < segmentInfo.dataInDiskSize || compressedBuffer.capacity() > segmentInfo.dataInDiskSize * 2) { + if (!Objects.isNull(compressedBuffer)) { + MmapUtil.clean((MappedByteBuffer) compressedBuffer); + } compressedBuffer = ByteBuffer.allocateDirect(segmentInfo.dataInDiskSize); } compressedBuffer.clear(); @@ -241,6 +250,9 @@ private void loadNextSegmentV2() throws IOException { if (Objects.isNull(dataBuffer) || dataBuffer.capacity() < segmentInfo.dataInDiskSize || dataBuffer.capacity() > segmentInfo.dataInDiskSize * 2) { + if (!Objects.isNull(dataBuffer)) { + MmapUtil.clean((MappedByteBuffer) dataBuffer); + } dataBuffer = ByteBuffer.allocateDirect(segmentInfo.dataInDiskSize); } dataBuffer.clear(); @@ -271,7 +283,14 @@ private void tryLoadSegment() throws IOException { } } - public void skipToGivenPosition(long pos) throws IOException { + /** + * Since current WAL file is compressed, but some part of the system need to skip the offset of an + * uncompressed wal file, this method is used to skip to the given logical position. + * + * @param pos The logical offset to skip to + * @throws IOException If the file is broken or the given position is invalid + */ + public void skipToGivenLogicalPosition(long pos) throws IOException { if (version == FileVersion.V2) { channel.position(WALWriter.MAGIC_STRING_BYTES); long posRemain = pos; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryPosition.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryPosition.java index 1e89d8c546d0..c794745c6f56 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryPosition.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/wal/utils/WALEntryPosition.java @@ -101,8 +101,9 @@ ByteBuffer read() throws IOException { if (!canRead()) { throw new IOException("Target file hasn't been specified."); } + // TODO: Reuse the file stream try (WALInputStream is = openReadFileStream()) { - is.skipToGivenPosition(position); + is.skipToGivenLogicalPosition(position); ByteBuffer buffer = ByteBuffer.allocate(size); is.read(buffer); buffer.flip(); @@ -137,6 +138,7 @@ public FileChannel openReadFileChannel() throws IOException { } public WALInputStream openReadFileStream() throws IOException { + // TODO: Refactor this part of code if (isInSealedFile()) { walFile = walNode.getWALFile(walFileVersionId); return new WALInputStream(walFile); diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/compression/WALCompressionTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/compression/WALCompressionTest.java index 7ab33343b5d5..d187f6107b6b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/compression/WALCompressionTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/wal/compression/WALCompressionTest.java @@ -150,7 +150,7 @@ public void testSkipToGivenPosition() try (WALInputStream stream = new WALInputStream(walFile)) { for (int i = 0; i < 100; ++i) { Pair positionAndNodePair = positionAndEntryPairList.get(i); - stream.skipToGivenPosition(positionAndNodePair.left); + stream.skipToGivenLogicalPosition(positionAndNodePair.left); /* Add the allocated buffer size by 2, because the actual serialized size of InsertRowNode is larger than the estimated value got by serializedSize. @@ -231,7 +231,7 @@ public void testCompressedWALStructure() } dataOutputStream.close(); ByteBuffer buf = ByteBuffer.wrap(baos.toByteArray()); - // Do not compress it + // Compress it IoTDBDescriptor.getInstance().getConfig().setWALCompressionAlgorithm(CompressionType.LZ4); WALTestUtils.setMinCompressionSize(0); try (WALWriter writer = new WALWriter(walFile)) { From d0b76f33de406a8206424d1eb149f1932693f320 Mon Sep 17 00:00:00 2001 From: Liu Xuxin Date: Mon, 17 Jun 2024 15:17:03 +0800 Subject: [PATCH 32/32] spotless --- .../src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java | 1 - 1 file changed, 1 deletion(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java index 1c324babca73..3e3fa348ed64 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/conf/IoTDBConfig.java @@ -3974,7 +3974,6 @@ public void setInnerCompactionTaskSelectionDiskRedundancy( this.innerCompactionTaskSelectionDiskRedundancy = innerCompactionTaskSelectionDiskRedundancy; } -<<<<<<< HEAD public TDataNodeLocation generateLocalDataNodeLocation() { TDataNodeLocation result = new TDataNodeLocation(); result.setDataNodeId(getDataNodeId());