diff --git a/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileGenerator.java b/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileGenerator.java index e0035353bf36..896745ad648c 100644 --- a/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileGenerator.java +++ b/integration-test/src/main/java/org/apache/iotdb/it/utils/TsFileGenerator.java @@ -228,7 +228,7 @@ private void generateTEXT(final Tablet tablet, final int column, final int row) public void generateDeletion(final String device, final int number) throws IOException, IllegalPathException { try (final ModificationFile modificationFile = - new ModificationFile(ModificationFile.getExclusiveMods(tsFile))) { + new ModificationFile(ModificationFile.getExclusiveMods(tsFile), false)) { writer.flush(); final TreeSet timeSet = device2TimeSet.get(device); if (timeSet.isEmpty()) { diff --git a/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileWithModIT.java b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileWithModIT.java new file mode 100644 index 000000000000..9d0a54f16728 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/db/it/IoTDBLoadTsFileWithModIT.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.it; + +import org.apache.iotdb.commons.exception.IllegalPathException; +import org.apache.iotdb.commons.path.MeasurementPath; +import org.apache.iotdb.db.exception.DataRegionException; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; +import org.apache.iotdb.db.storageengine.dataregion.modification.TreeDeletionEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Deletion; +import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.storageengine.dataregion.wal.recover.file.SealedTsFileRecoverPerformer; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.exception.write.WriteProcessException; +import org.apache.tsfile.write.TsFileWriter; +import org.apache.tsfile.write.record.Tablet; +import org.apache.tsfile.write.schema.MeasurementSchema; +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Collections; +import java.util.Objects; + +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class, ClusterIT.class}) +public class IoTDBLoadTsFileWithModIT { + private File tmpDir; + + @Before + public void setUp() throws Exception { + tmpDir = new File(Files.createTempDirectory("load").toUri()); + EnvFactory.getEnv().initClusterEnvironment(); + } + + @After + public void tearDown() throws Exception { + try { + for (final File file : Objects.requireNonNull(tmpDir.listFiles())) { + Files.delete(file.toPath()); + } + Files.delete(tmpDir.toPath()); + } finally { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + } + + private void generateFileWithNewModFile() + throws IOException, WriteProcessException, IllegalPathException, DataRegionException { + TsFileResource resource = generateFile(); + // write mods file + resource + .getExclusiveModFile() + .write(new TreeDeletionEntry(new MeasurementPath("root.test.d1.s1"), 1, 2)); + resource.getExclusiveModFile().close(); + } + + private void generateFileWithOldModFile() + throws IOException, DataRegionException, WriteProcessException, IllegalPathException { + TsFileResource resource = generateFile(); + ModificationFileV1 oldModFile = ModificationFileV1.getNormalMods(resource); + oldModFile.write(new Deletion(new MeasurementPath("root.test.d1.s1"), Long.MAX_VALUE, 1, 2)); + oldModFile.close(); + } + + private TsFileResource generateFile() + throws WriteProcessException, IOException, DataRegionException { + File tsfile = new File(tmpDir, "1-1-0-0.tsfile"); + try (TsFileWriter writer = new TsFileWriter(tsfile)) { + writer.registerAlignedTimeseries( + "root.test.d1", + Collections.singletonList(new MeasurementSchema("s1", TSDataType.BOOLEAN))); + Tablet tablet = + new Tablet( + "root.test.d1", + Collections.singletonList(new MeasurementSchema("s1", TSDataType.BOOLEAN))); + for (int i = 0; i < 5; i++) { + tablet.addTimestamp(i, i); + tablet.addValue(i, 0, true); + } + writer.writeTree(tablet); + } + // generate resource file + TsFileResource resource = new TsFileResource(tsfile); + try (SealedTsFileRecoverPerformer performer = new SealedTsFileRecoverPerformer(resource)) { + performer.recover(); + } + resource.setStatusForTest(TsFileResourceStatus.NORMAL); + resource.deserialize(); + return resource; + } + + @Test + public void testWithNewModFile() + throws SQLException, + IOException, + DataRegionException, + WriteProcessException, + IllegalPathException { + generateFileWithNewModFile(); + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + + statement.execute(String.format("load \'%s\'", tmpDir.getAbsolutePath())); + + try (final ResultSet resultSet = + statement.executeQuery("select count(s1) as c from root.test.d1")) { + Assert.assertTrue(resultSet.next()); + Assert.assertEquals(3, resultSet.getLong("c")); + } + } + } + + @Test + public void testWithOldModFile() + throws SQLException, + IOException, + DataRegionException, + WriteProcessException, + IllegalPathException { + generateFileWithOldModFile(); + try (final Connection connection = EnvFactory.getEnv().getConnection(); + final Statement statement = connection.createStatement()) { + + statement.execute(String.format("load \'%s\'", tmpDir.getAbsolutePath())); + + try (final ResultSet resultSet = + statement.executeQuery("select count(s1) as c from root.test.d1")) { + Assert.assertTrue(resultSet.next()); + Assert.assertEquals(3, resultSet.getLong("c")); + Assert.assertTrue( + new File(tmpDir, "1-1-0-0.tsfile" + ModificationFileV1.FILE_SUFFIX).exists()); + Assert.assertFalse( + new File(tmpDir, "1-1-0-0.tsfile" + ModificationFile.FILE_SUFFIX).exists()); + } + } + } +} diff --git a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java index ccd9ceedda88..4ca5c1fd2e2e 100644 --- a/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/session/it/IoTDBSessionSimpleIT.java @@ -600,6 +600,17 @@ public void insertTabletWithStringValuesTest() { while (dataSet.hasNext()) { RowRecord rowRecord = dataSet.next(); List fields = rowRecord.getFields(); + // this test may occasionally fail by IndexOutOfBoundsException + if (fields.size() != 7) { + SessionDataSet showTimeseriesDataSet = + session.executeQueryStatement("show timeseries root.sg1.d1.*"); + LOGGER.error("show timeseries result:"); + while (showTimeseriesDataSet.hasNext()) { + RowRecord row = showTimeseriesDataSet.next(); + LOGGER.error(row.toString()); + } + LOGGER.error("The number of fields is not correct. fields values: " + fields); + } assertEquals(fields.get(5).getBinaryV(), fields.get(6).getBinaryV()); } } catch (Exception e) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java index 776bc63d823c..5d6d2ced12ae 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTableSchemaCache.java @@ -33,6 +33,7 @@ import org.apache.iotdb.db.queryengine.plan.relational.metadata.Metadata; import org.apache.iotdb.db.queryengine.plan.relational.metadata.TableSchema; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; @@ -320,7 +321,7 @@ public void setCurrentModificationsAndTimeIndex( TsFileResource resource, TsFileSequenceReader reader) throws IOException { clearModificationsAndTimeIndex(); - currentModifications = resource.getAllModEntries(); + currentModifications = ModificationFile.readAllModifications(resource.getTsFile(), false); for (final ModEntry modification : currentModifications) { currentModificationsMemoryUsageSizeInBytes += modification.serializedSize(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java index ff9ea28224c4..6d50c14ecf7c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/analyze/load/LoadTsFileTreeSchemaCache.java @@ -25,6 +25,7 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.exception.load.LoadRuntimeOutOfMemoryException; import org.apache.iotdb.db.storageengine.dataregion.modification.ModEntry; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; @@ -153,7 +154,7 @@ public void setCurrentModificationsAndTimeIndex( TsFileResource resource, TsFileSequenceReader reader) throws IOException { clearModificationsAndTimeIndex(); - currentModifications = resource.getAllModEntries(); + currentModifications = ModificationFile.readAllModifications(resource.getTsFile(), true); for (final ModEntry modification : currentModifications) { currentModificationsMemoryUsageSizeInBytes += modification.serializedSize(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java index 84ff12e3dc7b..b343ced944ed 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/DataRegion.java @@ -3135,6 +3135,8 @@ private void loadModFile( final File newTargetModFile = ModificationFile.getExclusiveMods(targetTsFile); moveModFile(newModFileToLoad, newTargetModFile, deleteOriginFile); } + // force update mod file metrics + tsFileResource.getExclusiveModFile(); } @SuppressWarnings("java:S2139") diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java index c0f2807cf108..6d87b39a162c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/execute/task/InsertionCrossSpaceCompactionTask.java @@ -152,9 +152,6 @@ protected boolean doCompaction() { logger.logSourceFile(unseqFileToInsert); logger.logTargetFile(targetFile); logger.force(); - CompactionUtils.prepareCompactionModFiles( - Collections.singletonList(targetFile), Collections.singletonList(unseqFileToInsert)); - prepareTargetFiles(); validateCompactionResult( diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java index 2583e60e0ce4..c649b578b4c9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java @@ -23,6 +23,9 @@ import org.apache.iotdb.commons.path.PartialPath; import org.apache.iotdb.commons.utils.FileUtils; import org.apache.iotdb.db.service.metrics.FileMetrics; +import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Deletion; +import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Modification; +import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.slf4j.Logger; @@ -39,6 +42,7 @@ import java.nio.file.Files; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -68,24 +72,26 @@ public class ModificationFile implements AutoCloseable { private static final long COMPACT_THRESHOLD = 1024 * 1024L; private boolean hasCompacted = false; private boolean fileExists = false; + private final boolean updateMetrics; private Set cascadeFiles = null; - public ModificationFile(String filePath) { - this(new File(filePath)); + public ModificationFile(String filePath, boolean updateModMetrics) { + this(new File(filePath), updateModMetrics); } - public ModificationFile(File file) { + public ModificationFile(File file, boolean updateModMetrics) { this.file = file; fileExists = file.length() > 0; + this.updateMetrics = updateModMetrics; if (fileExists) { - FileMetrics.getInstance().increaseModFileNum(1); - FileMetrics.getInstance().increaseModFileSize(file.length()); + updateModFileMetric(1, file.length()); } } @SuppressWarnings("java:S2093") // cannot use try-with-resource, should not close here public void write(ModEntry entry) throws IOException { + int updateFileNum = 0; lock.writeLock().lock(); long size = 0; try { @@ -107,13 +113,14 @@ public void write(ModEntry entry) throws IOException { } if (!fileExists) { fileExists = true; - FileMetrics.getInstance().increaseModFileNum(1); + updateFileNum = 1; } - FileMetrics.getInstance().increaseModFileSize(size); + updateModFileMetric(updateFileNum, size); } @SuppressWarnings("java:S2093") // cannot use try-with-resource, should not close here public void write(Collection entries) throws IOException { + int updateFileNum = 0; lock.writeLock().lock(); long size = 0; try { @@ -136,10 +143,17 @@ public void write(Collection entries) throws IOException { lock.writeLock().unlock(); } if (!fileExists) { - FileMetrics.getInstance().increaseModFileNum(1); + updateFileNum = 1; fileExists = true; } - FileMetrics.getInstance().increaseModFileSize(size); + updateModFileMetric(updateFileNum, size); + } + + private void updateModFileMetric(int num, long size) { + if (updateMetrics) { + FileMetrics.getInstance().increaseModFileNum(num); + FileMetrics.getInstance().increaseModFileSize(size); + } } public Iterator getModIterator(long offset) throws IOException { @@ -199,6 +213,32 @@ public static long[] parseFileName(String name) { return new long[] {levelNum, modNum}; } + public static List readAllModifications( + File tsfile, boolean readOldModFileIfNewModFileNotExists) throws IOException { + try (ModificationFile modificationFile = + new ModificationFile(ModificationFile.getExclusiveMods(tsfile), false)) { + if (modificationFile.exists()) { + return modificationFile.getAllMods(); + } + } + if (!readOldModFileIfNewModFileNotExists) { + return Collections.emptyList(); + } + List result = new ArrayList<>(); + try (ModificationFileV1 modificationFileV1 = + new ModificationFileV1(ModificationFileV1.getNormalMods(tsfile).getPath())) { + if (!modificationFileV1.exists()) { + return Collections.emptyList(); + } + for (Modification modification : modificationFileV1.getModificationsIter()) { + if (modification instanceof Deletion) { + result.add(new TreeDeletionEntry((Deletion) modification)); + } + } + } + return result; + } + public class ModIterator implements Iterator, AutoCloseable { private InputStream inputStream; private ModEntry nextEntry; @@ -274,8 +314,9 @@ public boolean exists() { public void remove() throws IOException { close(); FileUtils.deleteFileOrDirectory(file); - FileMetrics.getInstance().decreaseModFileNum(1); - FileMetrics.getInstance().decreaseModFileSize(getFileLength()); + if (fileExists) { + updateModFileMetric(-1, -getFileLength()); + } fileExists = false; } @@ -284,7 +325,7 @@ public static ModificationFile getExclusiveMods(TsFileResource tsFileResource) { // replace the temp suffix with the final name tsFilePath = tsFilePath.replace(IoTDBConstant.INNER_COMPACTION_TMP_FILE_SUFFIX, TSFILE_SUFFIX); tsFilePath = tsFilePath.replace(IoTDBConstant.CROSS_COMPACTION_TMP_FILE_SUFFIX, TSFILE_SUFFIX); - return new ModificationFile(new File(tsFilePath + FILE_SUFFIX)); + return new ModificationFile(tsFilePath + FILE_SUFFIX, true); } public static File getExclusiveMods(File tsFile) { @@ -292,7 +333,8 @@ public static File getExclusiveMods(File tsFile) { } public static ModificationFile getCompactionMods(TsFileResource tsFileResource) { - return new ModificationFile(new File(tsFileResource.getTsFilePath() + COMPACTION_FILE_SUFFIX)); + return new ModificationFile( + new File(tsFileResource.getTsFilePath() + COMPACTION_FILE_SUFFIX), true); } public static File getCompactionMods(File tsFile) { @@ -318,7 +360,8 @@ public void compact() throws IOException { getAllMods().stream().collect(Collectors.groupingBy(ModEntry::keyOfPatternTree)); String newModsFileName = getFile().getPath() + COMPACT_SUFFIX; List allSettledModifications = new ArrayList<>(); - try (ModificationFile compactedModificationFile = new ModificationFile(newModsFileName)) { + try (ModificationFile compactedModificationFile = + new ModificationFile(newModsFileName, false)) { Set>> modificationsEntrySet = pathModificationMap.entrySet(); for (Map.Entry> modificationEntry : modificationsEntrySet) { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/PartitionLevelModFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/PartitionLevelModFileManager.java index c3fa4500ff15..592c839a1ffc 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/PartitionLevelModFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/PartitionLevelModFileManager.java @@ -50,7 +50,7 @@ public synchronized ModificationFile recover(String modFilePath, TsFileResource levelModFileIdMap.computeIfAbsent(level, l -> new TreeMap<>()); ModificationFile modificationFile = idModificationMap.computeIfAbsent( - modFileId, id -> new ModificationFile(new File(modFilePath))); + modFileId, id -> new ModificationFile(new File(modFilePath), true)); modFileReferences.computeIfAbsent(modificationFile, f -> new HashSet<>()).add(tsFileResource); @@ -128,7 +128,8 @@ private synchronized ModificationFile allocateNew(TsFileResource tsFileResource) new ModificationFile( new File( tsFileResource.getTsFile().getParentFile(), - ModificationFile.composeFileName(level, newId))); + ModificationFile.composeFileName(level, newId)), + true); idModificationMap.put(newId, newModFile); Set references = new HashSet<>(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java index 8627b663d032..e60d7e14ca1f 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/snapshot/SnapshotTaker.java @@ -222,7 +222,7 @@ private boolean createSnapshot(List resources, String snapshotId // rewrite the resource in snapshot to update the mod file path ModificationFile sharedModFileTemp = resource.getSharedModFile(); - resource.setSharedModFile(new ModificationFile(snapshotSharedModFile), false); + resource.setSharedModFile(new ModificationFile(snapshotSharedModFile, false), false); resource.serialize(snapshotResourceFile.getAbsolutePath()); resource.setSharedModFile(sharedModFileTemp, false); } else { diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java index 668679a8fe68..48ca35fbc7d9 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java @@ -313,7 +313,8 @@ public void deserialize() throws IOException { if (inputStream.available() > 0) { String modFilePath = ReadWriteIOUtils.readString(inputStream); - if (modFilePath != null && !modFilePath.isEmpty()) { + // ends with ".mods2" means it is a new version resource file + if (modFilePath != null && modFilePath.endsWith(ModificationFile.FILE_SUFFIX)) { sharedModFileOffset = ReadWriteIOUtils.readLong(inputStream); if (sharedModFilePathFuture != null) { sharedModFilePathFuture.complete(modFilePath); @@ -391,9 +392,11 @@ public void link(TsFileResource target) throws IOException { public void linkModFile(TsFileResource target) throws IOException { if (exclusiveModFileExists()) { + File modsFileForTargetResource = ModificationFile.getExclusiveMods(target.getTsFile()); Files.createLink( - ModificationFile.getExclusiveMods(target.getTsFile()).toPath(), + modsFileForTargetResource.toPath(), ModificationFile.getExclusiveMods(getTsFile()).toPath()); + target.setExclusiveModFile(new ModificationFile(modsFileForTargetResource, true)); } if (sharedModFileExists()) { modFileManagement.addReference(target, sharedModFile); @@ -1038,7 +1041,6 @@ public boolean isSpanMultiTimePartitions() { return timeIndex.isSpanMultiTimePartitions(); } - @TestOnly public void setExclusiveModFile(ModificationFile exclusiveModFile) { synchronized (this) { this.exclusiveModFile = exclusiveModFile; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java index b88ab2c65200..41bbbafe901c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/LoadTsFileManager.java @@ -487,7 +487,7 @@ private void writeDeletion(DataRegion dataRegion, DeletionData deletionData) } dataPartition2ModificationFile.put( - partitionInfo, new ModificationFile(newModificationFile.getAbsolutePath())); + partitionInfo, new ModificationFile(newModificationFile, false)); } ModificationFile modificationFile = dataPartition2ModificationFile.get(partitionInfo); writer.flush(); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java index 23ba30fde005..9252c5b0b90c 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/load/splitter/TsFileSplitter.java @@ -368,10 +368,7 @@ private void switchToTimeChunkContextOfCurrentMeasurement( } private void getAllModification(List deletions) throws IOException { - try (ModificationFile modificationFile = - new ModificationFile(ModificationFile.getExclusiveMods(tsFile))) { - deletions.addAll(modificationFile.getAllMods()); - } + deletions.addAll(ModificationFile.readAllModifications(tsFile, true)); } private boolean checkMagic(TsFileSequenceReader reader) throws IOException { diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java index a315285ad90f..d2468251a1f3 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/resource/PipeTsFileResourceManagerTest.java @@ -142,7 +142,7 @@ private void creatModsFile(String modsFilePath) throws IllegalPathException { new TreeDeletionEntry(new MeasurementPath("root.lemming.**"), 1, Long.MAX_VALUE) }; - try (ModificationFile mFile = new ModificationFile(modsFilePath)) { + try (ModificationFile mFile = new ModificationFile(new File(modsFilePath), false)) { for (ModEntry mod : modifications) { mFile.write(mod); } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSpaceCompactionSelectorTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSpaceCompactionSelectorTest.java index 52bab2209476..325901312b2a 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSpaceCompactionSelectorTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/compaction/inner/InnerSpaceCompactionSelectorTest.java @@ -715,7 +715,7 @@ class MockModiFicationFile extends ModificationFile { * @param filePath the path of the storage file. */ public MockModiFicationFile(String filePath) { - super(new File(filePath)); + super(new File(filePath), false); } @Override diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFileTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFileTest.java index f12b4fd35564..00053c54fca2 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFileTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFileTest.java @@ -64,7 +64,7 @@ public void readMyWrite() { new TimeRange(5, 6)), new TableDeletionEntry(new DeletionPredicate("table4"), new TimeRange(7, 8)), }; - try (ModificationFile mFile = new ModificationFile(tempFileName)) { + try (ModificationFile mFile = new ModificationFile(tempFileName, false)) { for (int i = 0; i < 4; i++) { mFile.write(modifications[i]); } @@ -105,7 +105,7 @@ public void writeVerifyTest() { new TimeRange(5, 6)), new TableDeletionEntry(new DeletionPredicate("table4"), new TimeRange(7, 8)), }; - try (ModificationFile mFile = new ModificationFile(tempFileName)) { + try (ModificationFile mFile = new ModificationFile(tempFileName, false)) { mFile.write(Arrays.asList(modifications)); List modificationList = mFile.getAllMods(); assertEquals(modifications.length, modificationList.size()); @@ -124,7 +124,7 @@ public void writeVerifyTest() { public void testCompact01() { String tempFileName = TestConstant.BASE_OUTPUT_PATH.concat("compact01.mods"); long time = 1000; - try (ModificationFile modificationFile = new ModificationFile(tempFileName)) { + try (ModificationFile modificationFile = new ModificationFile(tempFileName, false)) { while (modificationFile.getFileLength() < 1024 * 1024) { modificationFile.write( new TreeDeletionEntry( @@ -151,7 +151,7 @@ public void testCompact01() { public void testCompact02() { String tempFileName = TestConstant.BASE_OUTPUT_PATH.concat("compact02.mods"); long time = 1000; - try (ModificationFile modificationFile = new ModificationFile(tempFileName)) { + try (ModificationFile modificationFile = new ModificationFile(tempFileName, false)) { while (modificationFile.getFileLength() < 1024 * 100) { modificationFile.write( new TreeDeletionEntry( @@ -173,7 +173,7 @@ public void testCompact02() { @Test public void testCompact03() { String tempFileName = TestConstant.BASE_OUTPUT_PATH.concat("compact03.mods"); - try (ModificationFile modificationFile = new ModificationFile(tempFileName)) { + try (ModificationFile modificationFile = new ModificationFile(tempFileName, false)) { while (modificationFile.getFileLength() < 1024 * 1024) { modificationFile.write( new TreeDeletionEntry( @@ -198,7 +198,7 @@ public void testCompact03() { @Test public void testCompact04() { String tempFileName = TestConstant.BASE_OUTPUT_PATH.concat("compact04.mods"); - try (ModificationFile modificationFile = new ModificationFile(tempFileName)) { + try (ModificationFile modificationFile = new ModificationFile(tempFileName, false)) { long time = 0; while (modificationFile.getFileLength() < 1024 * 1024) { for (int i = 0; i < 5; i++) { @@ -228,7 +228,7 @@ public void testCompact04() { public void testCompact05() { String tempFileName = TestConstant.BASE_OUTPUT_PATH.concat("compact01.mods"); long time = 1000; - try (ModificationFile modificationFile = new ModificationFile(tempFileName)) { + try (ModificationFile modificationFile = new ModificationFile(tempFileName, false)) { while (modificationFile.getFileLength() < 1024 * 1024) { modificationFile.write( new TableDeletionEntry( @@ -253,7 +253,7 @@ public void testCompact05() { public void testCompact06() { String tempFileName = TestConstant.BASE_OUTPUT_PATH.concat("compact02.mods"); long time = 1000; - try (ModificationFile modificationFile = new ModificationFile(tempFileName)) { + try (ModificationFile modificationFile = new ModificationFile(tempFileName, false)) { while (modificationFile.getFileLength() < 1024 * 100) { modificationFile.write( new TableDeletionEntry( @@ -274,7 +274,7 @@ public void testCompact06() { @Test public void testCompact07() { String tempFileName = TestConstant.BASE_OUTPUT_PATH.concat("compact03.mods"); - try (ModificationFile modificationFile = new ModificationFile(tempFileName)) { + try (ModificationFile modificationFile = new ModificationFile(tempFileName, false)) { while (modificationFile.getFileLength() < 1024 * 1024) { modificationFile.write( new TableDeletionEntry( @@ -298,7 +298,7 @@ public void testCompact07() { @Test public void testCompact08() { String tempFileName = TestConstant.BASE_OUTPUT_PATH.concat("compact04.mods"); - try (ModificationFile modificationFile = new ModificationFile(tempFileName)) { + try (ModificationFile modificationFile = new ModificationFile(tempFileName, false)) { long time = 0; while (modificationFile.getFileLength() < 1024 * 1024) { for (int i = 0; i < 5; i++) { @@ -337,8 +337,8 @@ public void testRecover01() throws IOException { String modsFileName = TestConstant.BASE_OUTPUT_PATH.concat("compact01.mods"); String modsSettleFileName = TestConstant.BASE_OUTPUT_PATH.concat("compact01.mods.settle"); - try (ModificationFile modsFile = new ModificationFile(modsFileName); - ModificationFile modsSettleFile = new ModificationFile(modsSettleFileName)) { + try (ModificationFile modsFile = new ModificationFile(modsFileName, false); + ModificationFile modsSettleFile = new ModificationFile(modsSettleFileName, false)) { modsFile.write( new TreeDeletionEntry( @@ -369,7 +369,7 @@ public void testRecover01() throws IOException { public void testRecover02() throws IOException { String modsSettleFileName = TestConstant.BASE_OUTPUT_PATH.concat("compact02.mods.settle"); String originModsFileName = TestConstant.BASE_OUTPUT_PATH.concat("compact02.mods"); - try (ModificationFile modsSettleFile = new ModificationFile(modsSettleFileName)) { + try (ModificationFile modsSettleFile = new ModificationFile(modsSettleFileName, false)) { modsSettleFile.write( new TreeDeletionEntry( new MeasurementPath(new String[] {"root", "sg", "d1"}), @@ -393,8 +393,8 @@ public void testRecover03() throws IOException { String modsSettleFileName = TestConstant.BASE_OUTPUT_PATH.concat("compact01.mods.settle"); long time = 0; - try (ModificationFile modsFile = new ModificationFile(modsFileName); - ModificationFile modsSettleFile = new ModificationFile(modsSettleFileName)) { + try (ModificationFile modsFile = new ModificationFile(modsFileName, false); + ModificationFile modsSettleFile = new ModificationFile(modsSettleFileName, false)) { ModEntry[] modEntries = { new TableDeletionEntry( @@ -432,7 +432,7 @@ public void testRecover04() throws IOException { String originModsFileName = TestConstant.BASE_OUTPUT_PATH.concat("compact02.mods"); long time = 0; - try (ModificationFile modsSettleFile = new ModificationFile(modsSettleFileName)) { + try (ModificationFile modsSettleFile = new ModificationFile(modsSettleFileName, false)) { ModEntry[] modEntries = { new TableDeletionEntry( new DeletionPredicate("table1", new NOP()), diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/TsFileAndModSettleToolTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/TsFileAndModSettleToolTest.java index e49fa963ee45..92bd8e0c0ab6 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/TsFileAndModSettleToolTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/tools/TsFileAndModSettleToolTest.java @@ -166,7 +166,7 @@ private void createFile( } public void createlModificationFile(String timeseriesPath) { - ModificationFile modificationFile = new ModificationFile(new File(path)); + ModificationFile modificationFile = new ModificationFile(new File(path), false); List mods = new ArrayList<>(); try { MeasurementPath partialPath = new MeasurementPath(timeseriesPath);