From aeb877dcec2a48c7c9e7a29bdafcc12b2d4f6d3c Mon Sep 17 00:00:00 2001 From: JackieTien97 Date: Fri, 20 Nov 2026 17:44:20 +0800 Subject: [PATCH] Fix heap accounting for sealed TsFile resources Estimate resource metadata and device indexes separately. Track the charge for each registered file so metadata changes, degradation, and removal update the time-index budget consistently. --- .../plan/relational/sql/ast/LoadTsFile.java | 24 ++- .../schedule/CompactionScheduleContext.java | 8 +- .../modification/ModificationFile.java | 19 +- .../dataregion/tsfile/TsFileResource.java | 189 +++++++++++++---- .../timeindex/ArrayDeviceTimeIndex.java | 30 ++- .../tsfile/timeindex/FileTimeIndex.java | 2 +- .../rescon/memory/TsFileResourceManager.java | 194 +++++++++++++----- .../org/apache/iotdb/db/utils/MemUtils.java | 34 +++ .../rescon/memory/ResourceManagerTest.java | 137 ++++++++++++- 9 files changed, 518 insertions(+), 119 deletions(-) diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java index 2e9adadb5766e..8768305c4e1b5 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/plan/relational/sql/ast/LoadTsFile.java @@ -30,6 +30,7 @@ import org.apache.iotdb.db.queryengine.plan.statement.crud.LoadTsFileStatement; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.load.config.LoadTsFileConfigurator; +import org.apache.iotdb.db.utils.MemUtils; import org.apache.tsfile.utils.RamUsageEstimator; @@ -49,9 +50,6 @@ public class LoadTsFile extends Statement { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(LoadTsFile.class); - private static final long FILE_INSTANCE_SIZE = - RamUsageEstimator.shallowSizeOfInstance(File.class); - private String filePath; private int databaseLevel; // For loading to tree-model only @@ -389,14 +387,6 @@ public long ramBytesUsed() { size += RamUsageEstimator.sizeOf(filePath); size += RamUsageEstimator.sizeOf(database); size += RamUsageEstimator.sizeOfMap(loadAttributes); - if (tsFiles != null) { - size += RamUsageEstimator.shallowSizeOf(tsFiles); - for (File file : tsFiles) { - if (file != null) { - size += FILE_INSTANCE_SIZE; - } - } - } if (resources != null) { size += RamUsageEstimator.shallowSizeOf(resources); for (TsFileResource resource : resources) { @@ -405,6 +395,18 @@ public long ramBytesUsed() { } } } + if (tsFiles != null) { + size += RamUsageEstimator.shallowSizeOf(tsFiles); + for (int i = 0; i < tsFiles.size(); i++) { + File file = tsFiles.get(i); + // Analysis appends one resource (possibly null) for each processed file, in order. + TsFileResource resource = + resources != null && i < resources.size() ? resources.get(i) : null; + if (resource == null || resource.getTsFile() != file) { + size += MemUtils.getFileRamBytes(file); + } + } + } size += AstMemoryEstimationHelper.getEstimatedSizeOfLongList(writePointCountList); size += RamUsageEstimator.shallowSizeOf(isTableModel); return size; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleContext.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleContext.java index 6f0f7872789e6..b9fde5ab83abf 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleContext.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/compaction/schedule/CompactionScheduleContext.java @@ -89,9 +89,13 @@ public boolean isInsertionSelectionDelayed(long timePartitionId) { public void addResourceDeviceTimeIndex( TsFileResource tsFileResource, ArrayDeviceTimeIndex deviceTimeIndex) { - partitionFileDeviceInfoCache.put(tsFileResource, deviceTimeIndex); + ArrayDeviceTimeIndex previous = + partitionFileDeviceInfoCache.put(tsFileResource, deviceTimeIndex); + if (previous == deviceTimeIndex) { + return; + } long deviceTimeIndexSize = - tsFileResource.getDeviceTimeIndexRamSize().orElse(deviceTimeIndex.calculateRamSize()); + deviceTimeIndex.calculateRamSize() - (previous == null ? 0 : previous.calculateRamSize()); cachedDeviceInfoSize += deviceTimeIndexSize; CompactionMetrics.getInstance().addSelectionCachedDeviceTimeIndexSize(deviceTimeIndexSize); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java index 5ef17e9c01bc4..d9c62f02dd0c4 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/modification/ModificationFile.java @@ -28,7 +28,10 @@ import org.apache.iotdb.db.storageengine.dataregion.modification.v1.Modification; import org.apache.iotdb.db.storageengine.dataregion.modification.v1.ModificationFileV1; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; +import org.apache.iotdb.db.utils.MemUtils; +import org.apache.tsfile.utils.Accountable; +import org.apache.tsfile.utils.RamUsageEstimator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -51,6 +54,7 @@ import java.util.Map.Entry; import java.util.NoSuchElementException; import java.util.Set; +import java.util.concurrent.locks.AbstractQueuedSynchronizer; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.stream.Collectors; @@ -59,7 +63,14 @@ import static org.apache.iotdb.db.utils.ModificationUtils.sortAndMerge; import static org.apache.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX; -public class ModificationFile implements AutoCloseable { +public class ModificationFile implements AutoCloseable, Accountable { + + private static final long INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(ModificationFile.class) + + RamUsageEstimator.shallowSizeOfInstance(ReentrantReadWriteLock.class) + + RamUsageEstimator.shallowSizeOfInstance(ReentrantReadWriteLock.ReadLock.class) + + RamUsageEstimator.shallowSizeOfInstance(ReentrantReadWriteLock.WriteLock.class) + + RamUsageEstimator.shallowSizeOfInstance(AbstractQueuedSynchronizer.class); public static final String FILE_SUFFIX = ".mods2"; public static final String COMPACTION_FILE_SUFFIX = ".compaction.mods2"; @@ -92,6 +103,12 @@ public ModificationFile(File file, boolean updateModMetrics) { } } + /** Long-lived handle and lock. Open streams and cascaded files have separate lifetimes. */ + @Override + public long ramBytesUsed() { + return INSTANCE_SIZE + MemUtils.getFileRamBytes(file); + } + public void writeLock() { this.lock.writeLock().lock(); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java index 5629173986a91..f5f6e0f7061a2 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/TsFileResource.java @@ -48,8 +48,9 @@ import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.PlainDeviceTimeIndex; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.TimeIndexLevel; import org.apache.iotdb.db.storageengine.rescon.disk.TierManager; +import org.apache.iotdb.db.storageengine.rescon.memory.TsFileResourceManager.Registration; +import org.apache.iotdb.db.utils.MemUtils; import com.google.common.util.concurrent.RateLimiter; import org.apache.tsfile.file.metadata.IChunkMetadata; @@ -95,6 +96,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.atomic.AtomicReferenceFieldUpdater; import static org.apache.iotdb.commons.conf.IoTDBConstant.FILE_NAME_SEPARATOR; import static org.apache.tsfile.common.constant.TsFileConstant.TSFILE_SUFFIX; @@ -104,8 +106,17 @@ public class TsFileResource implements PersistentResource, Cloneable { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(TsFileResource.class) - + RamUsageEstimator.shallowSizeOfInstance(TsFileRepairStatus.class) + + RamUsageEstimator.shallowSizeOfInstance(TsFileLock.class) + + 3L * RamUsageEstimator.shallowSizeOfInstance(AtomicReference.class) + + RamUsageEstimator.shallowSizeOfInstance(AtomicBoolean.class) + TsFileID.SHALLOW_SIZE; + private static final long TIER_SIZE = + RamUsageEstimator.shallowSizeOfInstance(AtomicInteger.class); + private static final long PAIR_SIZE = RamUsageEstimator.shallowSizeOfInstance(Pair.class); + private static final long TIME_VALUE_PAIR_SIZE = + RamUsageEstimator.shallowSizeOfInstance(TimeValuePair.class); + private static final AtomicReferenceFieldUpdater TIME_INDEX_UPDATER = + AtomicReferenceFieldUpdater.newUpdater(TsFileResource.class, ITimeIndex.class, "timeIndex"); private static final Logger LOGGER = LoggerFactory.getLogger(TsFileResource.class); @@ -174,7 +185,7 @@ public class TsFileResource implements PersistentResource, Cloneable { private TsFileID tsFileID; - private long deviceTimeIndexRamSize; + private volatile Registration memoryRegistration; private AtomicInteger tierLevel; @@ -213,7 +224,7 @@ public class TsFileResource implements PersistentResource, Cloneable { private InsertionCompactionCandidateStatus insertionCompactionCandidateStatus = InsertionCompactionCandidateStatus.NOT_CHECKED; - private Map>> lastValues; + private volatile Map>> lastValues; @TestOnly public TsFileResource() { @@ -371,6 +382,8 @@ public void deserialize() throws IOException { break; } } + } finally { + updateMemoryCostIfRegistered(true); } } @@ -396,10 +409,12 @@ public void serializeFileTimeIndexToByteBuffer(ByteBuffer buffer) { public void updateStartTime(IDeviceID device, long time) { timeIndex.updateStartTime(device, time); + updateMemoryCostIfRegistered(true); } public void updateEndTime(IDeviceID device, long time) { timeIndex.updateEndTime(device, time); + updateMemoryCostIfRegistered(true); } public boolean resourceFileExists() { @@ -560,6 +575,7 @@ public ModificationFile getExclusiveModFile() { } else { exclusiveModFile = ModificationFile.getExclusiveMods(this); } + updateMemoryCostIfRegistered(); } } return exclusiveModFile; @@ -570,6 +586,7 @@ public ModificationFile getCompactionModFile() { synchronized (this) { if (compactionModFile == null) { compactionModFile = ModificationFile.getCompactionMods(this); + updateMemoryCostIfRegistered(); } } } @@ -580,7 +597,7 @@ public void removeCompactionModFile() throws IOException { if (compactionModFileExists() && !useSharedModFile) { getCompactionModFile().remove(); } - compactionModFile = null; + setCompactionModFile(null); } @TestOnly @@ -589,6 +606,7 @@ public void resetModFile() throws IOException { synchronized (this) { exclusiveModFile.close(); exclusiveModFile = null; + updateMemoryCostIfRegistered(); } } } @@ -596,6 +614,7 @@ public void resetModFile() throws IOException { public void setFile(File file) { this.file = file; this.tsFileID = new TsFileID(file.getAbsolutePath()); + updateMemoryCostIfRegistered(); } public File getTsFile() { @@ -794,6 +813,7 @@ public void closeWithoutSettingStatus() throws IOException { pathToReadOnlyMemChunkMap = null; pathToTimeSeriesMetadataMap = null; timeIndex.close(); + updateMemoryCostIfRegistered(true); } public TsFileProcessor getProcessor() { @@ -885,7 +905,7 @@ public boolean remove() { // Note that pipe can safely handle the case that the time index does not exist isEmpty(); if (getStatus() != TsFileResourceStatus.UNCLOSED) { - degradeTimeIndex(); + replaceDeviceTimeIndexWithFileTimeIndex(); } forceMarkDeleted(); try { @@ -1167,33 +1187,108 @@ public boolean isSpanMultiTimePartitions() { public void setExclusiveModFile(ModificationFile exclusiveModFile) { synchronized (this) { this.exclusiveModFile = exclusiveModFile; + updateMemoryCostIfRegistered(); } } - /** - * @return resource map size - */ + /** Estimated resident metadata of the file, excluding shared managers and query snapshot data. */ public long calculateRamSize() { - final ProgressIndex progressIndex = maxProgressIndex.get(); - if (timeIndex.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE) { - return INSTANCE_SIZE - + timeIndex.calculateRamSize() - + (Objects.nonNull(progressIndex) ? progressIndex.ramBytesUsed() : 0); + ITimeIndex index = timeIndex; + return calculateMetadataRamSize() + (index == null ? 0 : index.calculateRamSize()); + } + + /** The part that remains resident after a device index is replaced. */ + public long calculateMetadataRamSize() { + final ProgressIndex progress = maxProgressIndex.get(); + final ModificationFile exclusive = exclusiveModFile; + final ModificationFile compaction = compactionModFile; + long size = + INSTANCE_SIZE + + (tierLevel == null ? 0 : TIER_SIZE) + + MemUtils.getFileRamBytes(file) + + (progress == null || progress == MinimumProgressIndex.INSTANCE + ? 0 + : progress.ramBytesUsed()) + + (exclusive == null ? 0 : exclusive.ramBytesUsed()) + + (compaction == null || compaction == exclusive ? 0 : compaction.ramBytesUsed()); + final Future upgrade = exclusiveModFileFuture; + final CompletableFuture recovery = sharedModFilePathFuture; + if (upgrade != null) { + size += RamUsageEstimator.shallowSizeOf(upgrade); + } + if (recovery != null) { + size += RamUsageEstimator.shallowSizeOf(recovery); } - if (deviceTimeIndexRamSize == 0) { - deviceTimeIndexRamSize = timeIndex.calculateRamSize(); + if (originTsFileResource == null) { + // LOAD and asynchronous recovery can mark the file sealed without calling close(). + if (pathToChunkMetadataListMap != null) { + size += RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP; + } + if (pathToReadOnlyMemChunkMap != null) { + size += RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP; + } + if (pathToTimeSeriesMetadataMap != null) { + size += RamUsageEstimator.SHALLOW_SIZE_OF_CONCURRENT_HASHMAP; + } } - return INSTANCE_SIZE - + deviceTimeIndexRamSize - + (Objects.nonNull(progressIndex) ? progressIndex.ramBytesUsed() : 0); + return size + estimateLastValuesSize(lastValues); } - // used for compaction + private static long estimateLastValuesSize( + Map>> values) { + if (values == null) { + return 0; + } + long size = RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP; + for (Map.Entry>> entry : values.entrySet()) { + List> series = entry.getValue(); + size += + RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP_ENTRY + + RamUsageEstimator.NUM_BYTES_OBJECT_REF + + entry.getKey().ramBytesUsed() + + RamUsageEstimator.ARRAY_LIST_INSTANCE_SIZE + + RamUsageEstimator.sizeOfObjectArray(series.size()); + for (Pair pair : series) { + if (pair == null) { + continue; + } + size += PAIR_SIZE + RamUsageEstimator.sizeOf(pair.left); + if (pair.right != null) { + size += TIME_VALUE_PAIR_SIZE; + if (pair.right.getValue() != null) { + size += MemUtils.getTsPrimitiveTypeRamBytes(pair.right.getValue()); + } + } + } + } + return size; + } + + /** A rebuilt device index must be measured by its owner when the resident index is file-level. */ public Optional getDeviceTimeIndexRamSize() { - if (!this.isClosed()) { - return Optional.empty(); + ITimeIndex index = timeIndex; + return isClosed() && index instanceof ArrayDeviceTimeIndex + ? Optional.of(index.calculateRamSize()) + : Optional.empty(); + } + + public Registration getMemoryRegistration() { + return memoryRegistration; + } + + public void setMemoryRegistration(Registration registration) { + memoryRegistration = registration; + } + + private void updateMemoryCostIfRegistered() { + updateMemoryCostIfRegistered(false); + } + + private void updateMemoryCostIfRegistered(boolean indexChanged) { + Registration registration = memoryRegistration; + if (registration != null) { + registration.refresh(indexChanged); } - return Optional.of(deviceTimeIndexRamSize); } public long getMaxPlanIndex() { @@ -1273,6 +1368,7 @@ public TsFileID getTsFileID() { public void setTimeIndex(ITimeIndex timeIndex) { this.timeIndex = timeIndex; + updateMemoryCostIfRegistered(true); } /** @@ -1389,13 +1485,13 @@ public byte getTimeIndexType() { public void setTimeIndexType(byte type) { switch (type) { case ITimeIndex.ARRAY_DEVICE_TIME_INDEX_TYPE: - this.timeIndex = new ArrayDeviceTimeIndex(); + setTimeIndex(new ArrayDeviceTimeIndex()); break; case ITimeIndex.FILE_TIME_INDEX_TYPE: - this.timeIndex = new FileTimeIndex(); + setTimeIndex(new FileTimeIndex()); break; case ITimeIndex.PLAIN_DEVICE_TIME_INDEX_TYPE: - this.timeIndex = new PlainDeviceTimeIndex(); + setTimeIndex(new PlainDeviceTimeIndex()); break; default: throw new UnsupportedOperationException(); @@ -1404,19 +1500,23 @@ public void setTimeIndexType(byte type) { /** the DeviceTimeIndex degrade to FileTimeIndex and release memory */ public long degradeTimeIndex() { - TimeIndexLevel timeIndexLevel = TimeIndexLevel.valueOf(getTimeIndexType()); - // if current timeIndex is FileTimeIndex, no need to degrade - if (timeIndexLevel == TimeIndexLevel.FILE_TIME_INDEX) { - return 0; + ITimeIndex previous = replaceDeviceTimeIndexWithFileTimeIndex(); + return previous == null ? 0 : previous.calculateRamSize() - FileTimeIndex.INSTANCE_SIZE; + } + + /** Drops the device index without scanning it. The caller may measure the previous index. */ + public ITimeIndex replaceDeviceTimeIndexWithFileTimeIndex() { + ITimeIndex current = timeIndex; + while (current.getTimeIndexType() != ITimeIndex.FILE_TIME_INDEX_TYPE) { + FileTimeIndex replacement = + new FileTimeIndex(current.getMinStartTime(), current.getMaxEndTime()); + if (TIME_INDEX_UPDATER.compareAndSet(this, current, replacement)) { + updateMemoryCostIfRegistered(true); + return current; + } + current = timeIndex; } - // get the minimum startTime - long startTime = timeIndex.getMinStartTime(); - // get the maximum endTime - long endTime = timeIndex.getMaxEndTime(); - // replace the DeviceTimeIndex with FileTimeIndex - timeIndex = new FileTimeIndex(startTime, endTime); - // deviceTimeIndexRamSize has already been calculated before - return deviceTimeIndexRamSize - timeIndex.calculateRamSize(); + return null; } public void deleteRemovedDeviceAndUpdateEndTime(Map lastTimeForEachDevice) { @@ -1431,12 +1531,14 @@ public void deleteRemovedDeviceAndUpdateEndTime(Map lastTimeFor }); } timeIndex = newTimeIndex; + updateMemoryCostIfRegistered(true); } public void updateEndTime(Map lastTimeForEachDevice) { for (Map.Entry entry : lastTimeForEachDevice.entrySet()) { timeIndex.updateEndTime(entry.getKey(), entry.getValue()); } + updateMemoryCostIfRegistered(true); } /** @@ -1455,6 +1557,7 @@ public void updateProgressIndex(ProgressIndex progressIndex) { maxProgressIndex.updateAndGet( index -> index.updateToMinimumEqualOrIsAfterProgressIndex(progressIndex)); } + updateMemoryCostIfRegistered(); } public void setProgressIndex(ProgressIndex progressIndex) { @@ -1463,6 +1566,7 @@ public void setProgressIndex(ProgressIndex progressIndex) { } maxProgressIndex.set(progressIndex); + updateMemoryCostIfRegistered(); } @Override @@ -1610,6 +1714,7 @@ public void upgradeModFile(ExecutorService upgradeModFileThreadPool, boolean upd exclusiveModFileFuture = CompletableFuture.completedFuture(doUpgradeModFile(oldModFile, updateModMetrics)); } + updateMemoryCostIfRegistered(); } @SuppressWarnings({"java:S4042", "java:S899", "ResultOfMethodCallIgnored"}) @@ -1641,6 +1746,7 @@ public TsFileResource getNext() { public void setSharedModFilePathFuture(CompletableFuture sharedModFilePathFuture) { this.sharedModFilePathFuture = sharedModFilePathFuture; + updateMemoryCostIfRegistered(); } public boolean isUseSharedModFile() { @@ -1657,6 +1763,7 @@ public ModFileManagement getModFileManagement() { public void setCompactionModFile(ModificationFile compactionModFile) { this.compactionModFile = compactionModFile; + updateMemoryCostIfRegistered(); } public Map>> getLastValues() { @@ -1665,6 +1772,7 @@ public Map>> getLastValues() { public void setLastValues(Map>> lastValues) { this.lastValues = lastValues; + updateMemoryCostIfRegistered(); } public TsFileResource shallowClone() { @@ -1684,7 +1792,6 @@ public TsFileResource shallowClone() { cloned.isRecordedByMetric = this.isRecordedByMetric; cloned.tsFileRepairStatus = this.tsFileRepairStatus; cloned.settleTsFileCallBack = this.settleTsFileCallBack; - cloned.deviceTimeIndexRamSize = this.deviceTimeIndexRamSize; cloned.tsFileSize = this.tsFileSize; cloned.processor = this.processor; cloned.originTsFileResource = this.originTsFileResource; @@ -1706,6 +1813,8 @@ public TsFileResource shallowClone() { } public TsFileResource shallowCloneForNative() throws CloneNotSupportedException { - return (TsFileResource) clone(); + TsFileResource cloned = (TsFileResource) clone(); + cloned.memoryRegistration = null; + return cloned; } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java index 8af037a5924ef..9557cc3cfedc8 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/ArrayDeviceTimeIndex.java @@ -54,6 +54,14 @@ public class ArrayDeviceTimeIndex implements ITimeIndex { private static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(ArrayDeviceTimeIndex.class); + private static final long INTEGER_SIZE = RamUsageEstimator.shallowSizeOfInstance(Integer.class); + // Both ConcurrentHashMap.Node and HashMap.Node hold a hash and three references. + private static final long MAP_NODE_SIZE = + RamUsageEstimator.alignObjectSize( + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + + Integer.BYTES + + 3L * RamUsageEstimator.NUM_BYTES_OBJECT_REF); + private static final Logger logger = LoggerFactory.getLogger(ArrayDeviceTimeIndex.class); public static final int INIT_ARRAY_SIZE = 64; @@ -235,9 +243,27 @@ public boolean stillLives(long ttlLowerBound) { @Override public long calculateRamSize() { + int count = deviceToIndex.size(); + long deviceIdBytes = 0; + for (IDeviceID device : deviceToIndex.keySet()) { + deviceIdBytes += device.ramBytesUsed(); + } + // Capacity is an estimate: map implementations do not expose their backing array. + // ConcurrentHashMap grows at the threshold; HashMap grows on the next insertion. + boolean concurrent = deviceToIndex instanceof ConcurrentHashMap; + long requiredSlots = (4L * count - (concurrent ? 0 : 1)) / 3; + long buckets = + count == 0 ? 0 : Math.min(1L << 30, Math.max(16L, Long.highestOneBit(requiredSlots) << 1)); + long tableSize = buckets == 0 ? 0 : RamUsageEstimator.sizeOfObjectArray((int) buckets); return INSTANCE_SIZE - + RamUsageEstimator.sizeOfMap( - deviceToIndex, RamUsageEstimator.shallowSizeOfInstance(Integer.class)) + + (concurrent + ? RamUsageEstimator.SHALLOW_SIZE_OF_CONCURRENT_HASHMAP + : RamUsageEstimator.SHALLOW_SIZE_OF_HASHMAP) + + tableSize + + count * MAP_NODE_SIZE + + deviceIdBytes + // Indices are contiguous from zero. Integer.valueOf shares at least the first 128. + + Math.max(0, count - 128) * INTEGER_SIZE + RamUsageEstimator.sizeOf(startTimes) + RamUsageEstimator.sizeOf(endTimes); } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java index f6f598ef9a443..d96747e92bed6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/dataregion/tsfile/timeindex/FileTimeIndex.java @@ -48,7 +48,7 @@ public class FileTimeIndex implements ITimeIndex { - private static final long INSTANCE_SIZE = + public static final long INSTANCE_SIZE = RamUsageEstimator.shallowSizeOfInstance(FileTimeIndex.class); private static final Logger logger = LoggerFactory.getLogger(FileTimeIndex.class); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TsFileResourceManager.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TsFileResourceManager.java index 3221185db550e..733db3f504df6 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TsFileResourceManager.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/storageengine/rescon/memory/TsFileResourceManager.java @@ -26,11 +26,13 @@ import org.apache.iotdb.db.conf.IoTDBDescriptor; import org.apache.iotdb.db.i18n.StorageEngineMessages; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; -import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.TimeIndexLevel; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; +import org.apache.tsfile.utils.RamUsageEstimator; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.util.TreeSet; public class TsFileResourceManager { @@ -42,12 +44,60 @@ public class TsFileResourceManager { /** The memory block of time index */ private final IMemoryBlock memoryBlock; - /** store the sealed TsFileResource, sorted by priority of TimeIndex */ - private final TreeSet sealedTsFileResources = - new TreeSet<>(TsFileResource::compareIndexDegradePriority); + // One registration and one TreeMap.Entry (key, value, parent, left, right and color) per file. + static final long REGISTRATION_RAM_BYTES = + RamUsageEstimator.shallowSizeOfInstance(Registration.class) + + RamUsageEstimator.alignObjectSize( + RamUsageEstimator.NUM_BYTES_OBJECT_HEADER + + 5L * RamUsageEstimator.NUM_BYTES_OBJECT_REF + + Byte.BYTES); + + private final TreeSet sealedTsFileResources = new TreeSet<>(); + private long degradedTimeIndexNum; + private long nextRegistrationId; + private boolean degrading; + + /** The sort key and the charged bytes remain stable until the manager updates the entry. */ + public static final class Registration implements Comparable { + private final TsFileResource resource; + private final long id; + private long indexBytes; + private File file; + private long startTime; + private boolean fileIndex; + private long bytes; + + private Registration(TsFileResource resource, long id) { + this.resource = resource; + this.id = id; + updatePriority(resource.getTimeIndex()); + } + + private void updatePriority(ITimeIndex index) { + file = resource.getTsFile(); + startTime = index.getMinStartTime(); + fileIndex = index.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE; + } + + private int comparePriority(Registration other) { + int cmp = Boolean.compare(fileIndex, other.fileIndex); + if (cmp == 0) { + cmp = Long.compare(startTime, other.startTime); + } + return cmp != 0 ? cmp : file.getAbsolutePath().compareTo(other.file.getAbsolutePath()); + } + + @Override + public int compareTo(Registration other) { + int cmp = comparePriority(other); + // Two previously distinct entries can acquire the same priority after degradation. + return cmp == 0 ? Long.compare(id, other.id) : cmp; + } - // degraded time index number - private long degradedTimeIndexNum = 0; + public void refresh(boolean indexChanged) { + TsFileResourceManager.getInstance().refresh(this, indexChanged); + } + } private TsFileResourceManager() { memoryBlock = @@ -61,82 +111,115 @@ public void setTimeIndexMemoryThreshold(long timeIndexMemoryThreshold) { this.memoryBlock.setTotalMemorySizeInBytes(timeIndexMemoryThreshold); } - public long getPriorityQueueSize() { + public synchronized long getPriorityQueueSize() { return sealedTsFileResources.size(); } - /** - * add the closed TsFileResource into priorityQueue and increase memory cost of timeIndex, once - * memory cost is larger than threshold, degradation is triggered. - */ - public synchronized void registerSealedTsFileResource(TsFileResource tsFileResource) { - if (!sealedTsFileResources.contains(tsFileResource)) { - sealedTsFileResources.add(tsFileResource); - memoryBlock.forceAllocateWithoutLimitation(tsFileResource.calculateRamSize()); - chooseTsFileResourceToDegrade(); + public synchronized void registerSealedTsFileResource(TsFileResource resource) { + if (resource.getMemoryRegistration() != null) { + return; + } + Registration entry = new Registration(resource, ++nextRegistrationId); + Registration previous = sealedTsFileResources.floor(entry); + if (previous == null || entry.comparePriority(previous) != 0) { + sealedTsFileResources.add(entry); + if (entry.fileIndex) { + degradedTimeIndexNum++; + } + // Publish before sampling so a concurrent metadata change cannot miss its charge. + resource.setMemoryRegistration(entry); + try { + refresh(entry, true); + } catch (RuntimeException | Error e) { + removeTsFileResource(resource); + throw e; + } } } - /** delete the TsFileResource in PriorityQueue when the source file is deleted */ - public synchronized void removeTsFileResource(TsFileResource tsFileResource) { - if (sealedTsFileResources.contains(tsFileResource)) { - sealedTsFileResources.remove(tsFileResource); - if (TimeIndexLevel.valueOf(tsFileResource.getTimeIndexType()) - == TimeIndexLevel.FILE_TIME_INDEX) { - memoryBlock.release(tsFileResource.calculateRamSize()); + public synchronized void removeTsFileResource(TsFileResource resource) { + Registration entry = resource.getMemoryRegistration(); + if (entry == null) { + // Recovery can supply another resource describing the same file. + if (sealedTsFileResources.isEmpty()) { + return; + } + Registration key = new Registration(resource, Long.MIN_VALUE); + entry = sealedTsFileResources.ceiling(key); + if (entry == null || entry.comparePriority(key) != 0) { + return; + } + } + if (sealedTsFileResources.remove(entry)) { + entry.resource.setMemoryRegistration(null); + memoryBlock.release(entry.bytes); + if (entry.fileIndex) { degradedTimeIndexNum--; - } else { - memoryBlock.release(tsFileResource.calculateRamSize()); } } } - public void forceDegradeTsFileResource(TsFileResource resource) { - if (TimeIndexLevel.valueOf(resource.getTimeIndexType()) == TimeIndexLevel.FILE_TIME_INDEX) { + private synchronized void refresh(Registration entry, boolean indexChanged) { + TsFileResource resource = entry.resource; + if (resource.getMemoryRegistration() != entry) { return; } - logger.debug(StorageEngineMessages.FORCE_DEGRADE_TSFILE_RESOURCE, resource.getTsFilePath()); - synchronized (this) { - if (!sealedTsFileResources.remove(resource)) { - resource.degradeTimeIndex(); - return; + ITimeIndex index = resource.getTimeIndex(); + boolean fileIndex = index.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE; + if (entry.fileIndex != fileIndex + || entry.startTime != index.getMinStartTime() + || entry.file != resource.getTsFile()) { + sealedTsFileResources.remove(entry); + if (entry.fileIndex != fileIndex) { + degradedTimeIndexNum += fileIndex ? 1 : -1; } - long memoryReduce = resource.degradeTimeIndex(); - degradedTimeIndexNum++; - releaseTimeIndexMemCost(memoryReduce); - sealedTsFileResources.add(resource); + entry.updatePriority(index); + sealedTsFileResources.add(entry); + } + if (indexChanged) { + entry.indexBytes = index.calculateRamSize(); + } + long bytes = REGISTRATION_RAM_BYTES + resource.calculateMetadataRamSize() + entry.indexBytes; + long delta = bytes - entry.bytes; + entry.bytes = bytes; + if (delta > 0) { + memoryBlock.forceAllocateWithoutLimitation(delta); + if (!degrading + && memoryBlock.getUsedMemoryInBytes() > memoryBlock.getTotalMemorySizeInBytes()) { + degrading = true; + try { + chooseTsFileResourceToDegrade(); + } finally { + degrading = false; + } + } + } else if (delta < 0) { + memoryBlock.release(-delta); } } - /** once degradation is triggered, the total memory for timeIndex should reduce */ - private void releaseTimeIndexMemCost(long memCost) { - memoryBlock.release(memCost); + public void forceDegradeTsFileResource(TsFileResource resource) { + if (resource.replaceDeviceTimeIndexWithFileTimeIndex() != null) { + logger.debug(StorageEngineMessages.FORCE_DEGRADE_TSFILE_RESOURCE, resource.getTsFilePath()); + } } - /** - * choose the top TsFileResource in priorityQueue to degrade until the memory is smaller than - * threshold. - */ private void chooseTsFileResourceToDegrade() { while (memoryBlock.getUsedMemoryInBytes() > memoryBlock.getTotalMemorySizeInBytes()) { - TsFileResource tsFileResource = sealedTsFileResources.pollFirst(); - if (tsFileResource == null - || TimeIndexLevel.valueOf(tsFileResource.getTimeIndexType()) - == TimeIndexLevel.FILE_TIME_INDEX) { + Registration entry = sealedTsFileResources.first(); + if (entry.fileIndex) { logger.debug(StorageEngineMessages.CANNOT_DEGRADE_TIME_INDEX_ALL_FILE_LEVEL); - sealedTsFileResources.add(tsFileResource); return; } - long memoryReduce = tsFileResource.degradeTimeIndex(); - logger.debug(StorageEngineMessages.DEGRADE_TSFILE_RESOURCE, tsFileResource.getTsFilePath()); - degradedTimeIndexNum++; - releaseTimeIndexMemCost(memoryReduce); - // add the polled tsFileResource to the priority queue - sealedTsFileResources.add(tsFileResource); + logger.debug(StorageEngineMessages.DEGRADE_TSFILE_RESOURCE, entry.file); + if (entry.resource.replaceDeviceTimeIndexWithFileTimeIndex() == null) { + // Another thread has already swapped the index and is waiting to publish its size. + refresh(entry, true); + } } } - public long getDegradedTimeIndexNum() { + public synchronized long getDegradedTimeIndexNum() { return degradedTimeIndexNum; } @@ -151,6 +234,7 @@ public long getTotalTimeIndexMemCost() { /** function for clearing TsFileManager */ @TestOnly public synchronized void clear() { + sealedTsFileResources.forEach(entry -> entry.resource.setMemoryRegistration(null)); this.sealedTsFileResources.clear(); this.memoryBlock.setUsedMemoryInBytes(0); this.degradedTimeIndexNum = 0; diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java index b724c4d3c6f98..ad05bf6bc8c57 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/utils/MemUtils.java @@ -30,6 +30,7 @@ import org.apache.tsfile.enums.TSDataType; import org.apache.tsfile.utils.Binary; import org.apache.tsfile.utils.RamUsageEstimator; +import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.BooleanDataPoint; import org.apache.tsfile.write.record.datapoint.DataPoint; @@ -41,11 +42,44 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.util.List; // Notice : methods in this class may not be accurate. public class MemUtils { + private static final long FILE_INSTANCE_SIZE = + RamUsageEstimator.shallowSizeOfInstance(File.class); + + private static final ClassValue PRIMITIVE_INSTANCE_SIZE = + new ClassValue() { + @Override + protected Long computeValue(Class type) { + return RamUsageEstimator.shallowSizeOfInstance(type); + } + }; + + public static long getTsPrimitiveTypeRamBytes(TsPrimitiveType value) { + long bytes = PRIMITIVE_INSTANCE_SIZE.get(value.getClass()); + if (value instanceof TsPrimitiveType.TsBinary && value.getBinary() != null) { + bytes += value.getBinary().ramBytesUsed(); + } else if (value instanceof TsPrimitiveType.TsVector && value.getVector() != null) { + TsPrimitiveType[] elements = value.getVector(); + bytes += RamUsageEstimator.shallowSizeOf(elements); + for (TsPrimitiveType element : elements) { + if (element != null) { + bytes += getTsPrimitiveTypeRamBytes(element); + } + } + } + return bytes; + } + + /** Heap occupied by a file handle and its stored path. */ + public static long getFileRamBytes(File file) { + return file == null ? 0 : FILE_INSTANCE_SIZE + RamUsageEstimator.sizeOf(file.getPath()); + } + private static final Logger LOGGER = LoggerFactory.getLogger(MemUtils.class); private MemUtils() {} diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/ResourceManagerTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/ResourceManagerTest.java index 4236cbcd576af..8926999c3f17e 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/ResourceManagerTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/storageengine/rescon/memory/ResourceManagerTest.java @@ -20,6 +20,7 @@ package org.apache.iotdb.db.storageengine.rescon.memory; import org.apache.iotdb.commons.conf.IoTDBConstant; +import org.apache.iotdb.commons.consensus.index.impl.SimpleProgressIndex; import org.apache.iotdb.commons.exception.MetadataException; import org.apache.iotdb.db.conf.DataNodeMemoryConfig; import org.apache.iotdb.db.conf.IoTDBConfig; @@ -31,9 +32,12 @@ import org.apache.iotdb.db.storageengine.dataregion.read.control.FileReaderManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ArrayDeviceTimeIndex; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.ITimeIndex; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.TimeIndexLevel; import org.apache.iotdb.db.utils.EnvironmentUtils; +import org.apache.iotdb.db.utils.MemUtils; import org.apache.iotdb.db.utils.constant.TestConstant; import org.apache.tsfile.enums.TSDataType; @@ -42,7 +46,11 @@ import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.apache.tsfile.fileSystem.FSFactoryProducer; +import org.apache.tsfile.read.TimeValuePair; import org.apache.tsfile.read.common.Path; +import org.apache.tsfile.utils.Binary; +import org.apache.tsfile.utils.Pair; +import org.apache.tsfile.utils.TsPrimitiveType; import org.apache.tsfile.write.TsFileWriter; import org.apache.tsfile.write.record.TSRecord; import org.apache.tsfile.write.record.datapoint.DataPoint; @@ -56,10 +64,13 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import static org.apache.iotdb.commons.conf.IoTDBConstant.PATH_SEPARATOR; +import static org.apache.iotdb.db.storageengine.rescon.memory.TsFileResourceManager.REGISTRATION_RAM_BYTES; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; public class ResourceManagerTest { @@ -246,7 +257,7 @@ public void testNotDegradeToFileTimeIndex() throws IOException, WriteProcessExce TimeIndexLevel.ARRAY_DEVICE_TIME_INDEX, TimeIndexLevel.valueOf(tsFileResource.getTimeIndexType())); long previousRamSize = tsFileResource.calculateRamSize(); - long curTimeIndexMemoryThreshold = 4291; + long curTimeIndexMemoryThreshold = previousRamSize + REGISTRATION_RAM_BYTES; tsFileResourceManager.setTimeIndexMemoryThreshold(curTimeIndexMemoryThreshold); tsFileResourceManager.registerSealedTsFileResource(tsFileResource); assertEquals(0, previousRamSize - tsFileResource.calculateRamSize()); @@ -302,7 +313,9 @@ public void testTwoResourceToDegrade() throws IOException, WriteProcessException TimeIndexLevel.valueOf(tsFileResource2.getTimeIndexType())); long curTimeIndexMemoryThreshold = - degradedResource1.calculateRamSize() + tsFileResource2.calculateRamSize(); + degradedResource1.calculateRamSize() + + tsFileResource2.calculateRamSize() + + 2 * REGISTRATION_RAM_BYTES; tsFileResourceManager.setTimeIndexMemoryThreshold(curTimeIndexMemoryThreshold); tsFileResourceManager.registerSealedTsFileResource(tsFileResource1); assertEquals( @@ -340,7 +353,7 @@ public void testMultiDeviceTimeIndexDegrade() throws IOException, WriteProcessEx TimeIndexLevel.valueOf(tsFileResource.getTimeIndexType())); prepareFile(tsFileResource, i * ptNum, ptNum, 0); tsFileResource.degradeTimeIndex(); - curTimeIndexMemoryThreshold += tsFileResource.calculateRamSize(); + curTimeIndexMemoryThreshold += tsFileResource.calculateRamSize() + REGISTRATION_RAM_BYTES; } for (int i = 8; i < seqFileNum; i++) { File file = @@ -361,7 +374,7 @@ public void testMultiDeviceTimeIndexDegrade() throws IOException, WriteProcessEx TimeIndexLevel.ARRAY_DEVICE_TIME_INDEX, TimeIndexLevel.valueOf(tsFileResource.getTimeIndexType())); prepareFile(tsFileResource, i * ptNum, ptNum, 0); - curTimeIndexMemoryThreshold += tsFileResource.calculateRamSize(); + curTimeIndexMemoryThreshold += tsFileResource.calculateRamSize() + REGISTRATION_RAM_BYTES; } for (int i = 0; i < seqFileNum; i++) { @@ -459,13 +472,123 @@ public void testForceDegradeTimeIndex() { tsFileResourceManager.registerSealedTsFileResource(tsFileResource); long resourceRamSizeBeforeDegrade = tsFileResource.calculateRamSize(); Assert.assertEquals( - resourceRamSizeBeforeDegrade, tsFileResourceManager.getTotalTimeIndexMemCost()); + resourceRamSizeBeforeDegrade + REGISTRATION_RAM_BYTES, + tsFileResourceManager.getTotalTimeIndexMemCost()); TsFileResourceManager.getInstance().forceDegradeTsFileResource(tsFileResource); Assert.assertEquals( - tsFileResource.calculateRamSize(), tsFileResourceManager.getTotalTimeIndexMemCost()); + tsFileResource.calculateRamSize() + REGISTRATION_RAM_BYTES, + tsFileResourceManager.getTotalTimeIndexMemCost()); Assert.assertTrue( - resourceRamSizeBeforeDegrade > tsFileResourceManager.getTotalTimeIndexMemCost()); + resourceRamSizeBeforeDegrade + REGISTRATION_RAM_BYTES + > tsFileResourceManager.getTotalTimeIndexMemCost()); Assert.assertTrue(tsFileResource.getTimeIndexType() == ITimeIndex.FILE_TIME_INDEX_TYPE); Assert.assertEquals(1, tsFileResourceManager.getDegradedTimeIndexNum()); } + + @Test + public void testResidentMetadataAndIndexSizeChanges() throws IOException { + File file = new File("target/0-1-0-0.tsfile"); + TsFileResource resource = new TsFileResource(file); + long before = resource.calculateRamSize(); + IDeviceID device = IDeviceID.Factory.DEFAULT_FACTORY.create("root.test.newDevice"); + resource.updateStartTime(device, 1); + assertTrue(resource.calculateRamSize() > before); + + resource.close(); + long sealed = resource.calculateRamSize(); + long indexSize = resource.getTimeIndex().calculateRamSize(); + assertEquals(Long.valueOf(indexSize), resource.getDeviceTimeIndexRamSize().get()); + long expectedRelease = indexSize - new FileTimeIndex().calculateRamSize(); + // A freshly sealed resource can also be degraded before any size calculation. + TsFileResource unmeasured = new TsFileResource(file); + unmeasured.updateStartTime(device, 1); + unmeasured.close(); + assertEquals(expectedRelease, unmeasured.degradeTimeIndex()); + assertEquals(expectedRelease, resource.degradeTimeIndex()); + assertEquals(sealed - expectedRelease, resource.calculateRamSize()); + assertEquals(0, resource.degradeTimeIndex()); + Assert.assertFalse(resource.getDeviceTimeIndexRamSize().isPresent()); + + long base = resource.calculateRamSize(); + File longerPath = new File("target/longer/directory/0-1-0-0.tsfile"); + resource.setFile(longerPath); + assertEquals( + base + MemUtils.getFileRamBytes(longerPath) - MemUtils.getFileRamBytes(file), + resource.calculateRamSize()); + long beforeMods = resource.calculateRamSize(); + long modSize = resource.getExclusiveModFile().ramBytesUsed(); + assertEquals(beforeMods + modSize, resource.calculateRamSize()); + resource.resetModFile(); + assertEquals(beforeMods, resource.calculateRamSize()); + } + + @Test + public void testRegisteredBytesFollowMetadataAndIndexChanges() throws Exception { + tsFileResourceManager.setTimeIndexMemoryThreshold(Long.MAX_VALUE); + TsFileResource remaining = new TsFileResource(new File("target/0-1-0-0.tsfile")); + remaining.close(); + tsFileResourceManager.registerSealedTsFileResource(remaining); + long baseline = tsFileResourceManager.getTotalTimeIndexMemCost(); + TsFileResource resource = new TsFileResource(new File("target/0-2-0-0.tsfile")); + resource.close(); + resource.setTimeIndex(new ArrayDeviceTimeIndex()); + tsFileResourceManager.registerSealedTsFileResource(resource); + tsFileResourceManager.registerSealedTsFileResource(resource); + long originalCharge = resource.calculateRamSize() + REGISTRATION_RAM_BYTES; + assertEquals(baseline + originalCharge, tsFileResourceManager.getTotalTimeIndexMemCost()); + resource.updateStartTime(IDeviceID.Factory.DEFAULT_FACTORY.create("root.test.later"), 7); + resource.setProgressIndex(new SimpleProgressIndex(1, 1)); + resource.getExclusiveModFile(); + long changedCharge = resource.calculateRamSize() + REGISTRATION_RAM_BYTES; + assertTrue(changedCharge > originalCharge); + assertEquals(baseline + changedCharge, tsFileResourceManager.getTotalTimeIndexMemCost()); + + resource.degradeTimeIndex(); + assertEquals(1, tsFileResourceManager.getDegradedTimeIndexNum()); + assertEquals( + baseline + resource.calculateRamSize() + REGISTRATION_RAM_BYTES, + tsFileResourceManager.getTotalTimeIndexMemCost()); + TsFileResource clone = resource.shallowCloneForNative(); + Assert.assertNull(clone.getMemoryRegistration()); + clone.getCompactionModFile(); + assertEquals( + baseline + resource.calculateRamSize() + REGISTRATION_RAM_BYTES, + tsFileResourceManager.getTotalTimeIndexMemCost()); + resource.setTimeIndex(new ArrayDeviceTimeIndex()); + assertEquals(0, tsFileResourceManager.getDegradedTimeIndexNum()); + assertEquals( + baseline + resource.calculateRamSize() + REGISTRATION_RAM_BYTES, + tsFileResourceManager.getTotalTimeIndexMemCost()); + tsFileResourceManager.removeTsFileResource(resource); + tsFileResourceManager.removeTsFileResource(resource); + assertEquals(baseline, tsFileResourceManager.getTotalTimeIndexMemCost()); + assertEquals(0, tsFileResourceManager.getDegradedTimeIndexNum()); + tsFileResourceManager.removeTsFileResource(remaining); + assertEquals(0, tsFileResourceManager.getTotalTimeIndexMemCost()); + } + + @Test + public void testLoadLastValuesReleasedFromRegisteredBytes() throws IOException { + tsFileResourceManager.setTimeIndexMemoryThreshold(Long.MAX_VALUE); + TsFileResource resource = new TsFileResource(new File("target/0-3-0-0.tsfile")); + resource.setTimeIndex(new FileTimeIndex(1, 2)); + resource.close(); + tsFileResourceManager.registerSealedTsFileResource(resource); + assertEquals(1, tsFileResourceManager.getDegradedTimeIndexNum()); + long baseline = tsFileResourceManager.getTotalTimeIndexMemCost(); + TsPrimitiveType.TsBinary value = new TsPrimitiveType.TsBinary(new Binary(new byte[1024])); + resource.setLastValues( + Collections.singletonMap( + IDeviceID.Factory.DEFAULT_FACTORY.create("root.test.d"), + Collections.singletonList(new Pair<>("s1", new TimeValuePair(1, value))))); + assertTrue(tsFileResourceManager.getTotalTimeIndexMemCost() >= baseline + 1024); + assertEquals( + resource.calculateRamSize() + REGISTRATION_RAM_BYTES, + tsFileResourceManager.getTotalTimeIndexMemCost()); + resource.setLastValues(null); + assertEquals(baseline, tsFileResourceManager.getTotalTimeIndexMemCost()); + tsFileResourceManager.removeTsFileResource(resource); + assertEquals(0, tsFileResourceManager.getDegradedTimeIndexNum()); + assertEquals(0, tsFileResourceManager.getTotalTimeIndexMemCost()); + } }