diff --git a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index 3153e962e115..d0312c0fb953 100644 --- a/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/en/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -651,6 +651,9 @@ public final class DataNodePipeMessages { "Pipe {}@{}: failed to unpin TsFileResource after creating event, original path: {}"; public static final String PIPE_FAILED_TO_UNPIN_TSFILERESOURCE_AFTER_DROPPING = "Pipe {}@{}: failed to unpin TsFileResource after dropping pipe, original path: {}"; + public static final String + EXCEPTION_PIPE_ARG_ARG_TIME_PARTITION_ARG_CHANGED_DURING_HISTORICAL_TSFILE_EXTRACTION_AFTER_ONE_REFRESH_AF060BD3 = + "Pipe {}@{}: time partition {} changed during historical TsFile extraction after one refresh"; public static final String PIPE_FINISH_TO_EXTRACT_DELETIONS_EXTRACT_DELETIONS = "Pipe {}@{}: finish to extract deletions, extract deletions count {}/{}, took {} ms"; public static final String PIPE_FINISH_TO_EXTRACT_HISTORICAL_TSFILE_EXTRACTED = diff --git a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java index 550323119fdc..f1feff5d97ae 100644 --- a/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java +++ b/iotdb-core/datanode/src/main/i18n/zh/org/apache/iotdb/db/i18n/DataNodePipeMessages.java @@ -617,6 +617,9 @@ public final class DataNodePipeMessages { "Pipe {}@{}:unpin TsFileResource after creating event, original path: {} 失败"; public static final String PIPE_FAILED_TO_UNPIN_TSFILERESOURCE_AFTER_DROPPING = "Pipe {}@{}:unpin TsFileResource after dropping pipe, original path: {} 失败"; + public static final String + EXCEPTION_PIPE_ARG_ARG_TIME_PARTITION_ARG_CHANGED_DURING_HISTORICAL_TSFILE_EXTRACTION_AFTER_ONE_REFRESH_AF060BD3 = + "Pipe {}@{}:历史 TsFile 提取期间,时间分区 {} 在刷新一次后仍发生变化"; public static final String PIPE_FINISH_TO_EXTRACT_DELETIONS_EXTRACT_DELETIONS = "Pipe {}@{}:finish to extract deletions, extract deletions count {}/{}, took {} ms"; public static final String PIPE_FINISH_TO_EXTRACT_HISTORICAL_TSFILE_EXTRACTED = diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java index 9940127488ad..ed8f5e84406a 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/IoTDBDataRegionSource.java @@ -42,6 +42,7 @@ import org.apache.iotdb.db.protocol.session.InternalClientSession; import org.apache.iotdb.db.protocol.session.SessionManager; import org.apache.iotdb.db.storageengine.StorageEngine; +import org.apache.iotdb.db.storageengine.dataregion.DataRegion; import org.apache.iotdb.pipe.api.annotation.TableModel; import org.apache.iotdb.pipe.api.annotation.TreeModel; import org.apache.iotdb.pipe.api.customizer.configuration.PipeExtractorRuntimeConfiguration; @@ -552,20 +553,12 @@ public void start() throws Exception { .runIfPresent( dataRegionIdObject, (dataRegion -> { - dataRegion.writeLock( - String.format( - DataNodePipeMessages.MESSAGE_PIPE_STARTING_ARG_A09517B7, - IoTDBDataRegionSource.class.getName())); - try { - startHistoricalExtractorAndRealtimeExtractor(exceptionHolder); - } finally { - dataRegion.writeUnlock(); - } + startHistoricalExtractorAndRealtimeExtractor(exceptionHolder, dataRegion); })) || StorageEngine.getInstance() .runIfAbsent( dataRegionIdObject, - () -> startHistoricalExtractorAndRealtimeExtractor(exceptionHolder))) { + () -> startHistoricalExtractorAndRealtimeExtractor(exceptionHolder, null))) { rethrowExceptionIfAny(exceptionHolder); completionSourceStarted.set(true); @@ -583,14 +576,26 @@ public void start() throws Exception { } private void startHistoricalExtractorAndRealtimeExtractor( - final AtomicReference exceptionHolder) { + final AtomicReference exceptionHolder, final DataRegion dataRegion) { try { // Start realtimeSource first to avoid losing data. This may cause some // retransmission, yet it is OK according to the idempotency of IoTDB. // Note: The order of historical collection is flushing data -> adding all tsFile events. // There can still be writing when tsFile events are added. If we start // realtimeSource after the process, then this part of data will be lost. - realtimeSource.start(); + if (Objects.nonNull(dataRegion)) { + dataRegion.writeLock( + String.format( + DataNodePipeMessages.MESSAGE_PIPE_STARTING_ARG_A09517B7, + IoTDBDataRegionSource.class.getName())); + try { + realtimeSource.start(); + } finally { + dataRegion.writeUnlock(); + } + } else { + realtimeSource.start(); + } historicalSource.start(); } catch (final Exception e) { exceptionHolder.set(e); diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java index bc34b4acd01e..766b4cfff675 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSource.java @@ -77,6 +77,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.File; import java.io.IOException; import java.util.ArrayDeque; import java.util.ArrayList; @@ -90,7 +91,7 @@ import java.util.Optional; import java.util.Queue; import java.util.Set; -import java.util.function.Function; +import java.util.TreeMap; import java.util.stream.Collectors; import static org.apache.iotdb.commons.pipe.config.constant.PipeSourceConstant.EXTRACTOR_END_TIME_KEY; @@ -177,6 +178,7 @@ public class PipeHistoricalDataRegionTsFileAndDeletionSource private Queue pendingQueue; private final Map> filteredTsFileResources2TableNames = new HashMap<>(); + private final Set resourcesWithPinnedTsFile = new HashSet<>(); private final Set resourcesWithPinnedModFile = new HashSet<>(); private final Map pendingResource2ReplicateIndexForIoTV2 = new HashMap<>(); @@ -564,6 +566,10 @@ public synchronized void start() { } final long startHistoricalExtractionTime = System.currentTimeMillis(); + final TsFileManager tsFileManager = dataRegion.getTsFileManager(); + List sequenceTsFileResourceSnapshot = Collections.emptyList(); + List unsequenceTsFileResourceSnapshot = Collections.emptyList(); + List deletionResourceSnapshot = Collections.emptyList(); dataRegion.writeLock( DataNodePipeMessages .MESSAGE_PIPE_START_TO_EXTRACT_HISTORICAL_TSFILE_AND_DELETION_IF_USES_IOTCONSENSUSV2_C1603968); @@ -572,44 +578,65 @@ public synchronized void start() { pendingQueue = new ArrayDeque<>(); return; } - List originalResourceList = new ArrayList<>(); - if (shouldExtractInsertion) { flushTsFilesForExtraction(dataRegion); - extractTsFiles(dataRegion, startHistoricalExtractionTime, originalResourceList); + tsFileManager.readLock(); + try { + final Pair, List> snapshot = + tsFileManager.getAllTsFileListForQuery(null, null); + sequenceTsFileResourceSnapshot = snapshot.left; + unsequenceTsFileResourceSnapshot = snapshot.right; + } finally { + tsFileManager.readUnlock(); + } } if (shouldExtractDeletion) { - Optional.ofNullable(DeletionResourceManager.getInstance(dataRegionId)) - .ifPresent(manager -> extractDeletions(manager, originalResourceList)); + deletionResourceSnapshot = + Optional.ofNullable(DeletionResourceManager.getInstance(dataRegionId)) + .map(DeletionResourceManager::getAllDeletionResources) + .orElse(Collections.emptyList()); } - - // Sort tsFileResource and deletionResource - long startTime = System.currentTimeMillis(); - LOGGER.info( - DataNodePipeMessages.PIPE_START_TO_SORT_ALL_EXTRACTED_RESOURCES, pipeName, dataRegionId); - if (shouldUseHistoricalTsFileQueryPriorityOrder()) { - prepareResourcesForHistoricalTsFileQueryPriorityOrder(originalResourceList); - } - sortExtractedResources(originalResourceList); - if (shouldUseHistoricalTsFileQueryPriorityOrder()) { - prepareProgressReportResourcesForHistoricalTsFileQueryPriorityOrder(originalResourceList); - } - pendingQueue = new ArrayDeque<>(originalResourceList); - PipeTerminateEvent.initializeHistoricalTransferSummary( - pipeName, - creationTime, - dataRegionId, - extractedHistoricalTsFileCount, - extractedHistoricalDeletionCount); - - LOGGER.info( - DataNodePipeMessages.PIPE_FINISH_TO_SORT_ALL_EXTRACTED_RESOURCES, - pipeName, - dataRegionId, - System.currentTimeMillis() - startTime); } finally { dataRegion.writeUnlock(); } + + final List originalResourceList = new ArrayList<>(); + if (shouldExtractInsertion) { + extractTsFiles( + tsFileManager, + sequenceTsFileResourceSnapshot, + unsequenceTsFileResourceSnapshot, + startHistoricalExtractionTime, + originalResourceList); + } + if (shouldExtractDeletion) { + extractDeletions(deletionResourceSnapshot, originalResourceList); + } + + // Sort tsFileResource and deletionResource + long startTime = System.currentTimeMillis(); + LOGGER.info( + DataNodePipeMessages.PIPE_START_TO_SORT_ALL_EXTRACTED_RESOURCES, pipeName, dataRegionId); + if (shouldUseHistoricalTsFileQueryPriorityOrder()) { + prepareResourcesForHistoricalTsFileQueryPriorityOrder(originalResourceList); + } + sortExtractedResources(originalResourceList); + if (shouldUseHistoricalTsFileQueryPriorityOrder()) { + prepareProgressReportResourcesForHistoricalTsFileQueryPriorityOrder(originalResourceList); + } + pendingQueue = new ArrayDeque<>(originalResourceList); + PipeTerminateEvent.initializeHistoricalTransferSummary( + pipeName, + creationTime, + dataRegionId, + extractedHistoricalTsFileCount, + extractedHistoricalDeletionCount); + + LOGGER.info( + DataNodePipeMessages.PIPE_FINISH_TO_SORT_ALL_EXTRACTED_RESOURCES, + pipeName, + dataRegionId, + System.currentTimeMillis() - startTime); } private boolean shouldUseHistoricalTsFileQueryPriorityOrder() { @@ -753,108 +780,328 @@ private void flushTsFilesForExtraction(DataRegion dataRegion) { } private void extractTsFiles( - final DataRegion dataRegion, + final TsFileManager tsFileManager, + final List sequenceTsFileResourceSnapshot, + final List unsequenceTsFileResourceSnapshot, final long startHistoricalExtractionTime, final List originalResourceList) { - final TsFileManager tsFileManager = dataRegion.getTsFileManager(); - tsFileManager.readLock(); + final int originalSequenceTsFileCount = sequenceTsFileResourceSnapshot.size(); + final int originalUnSequenceTsFileCount = unsequenceTsFileResourceSnapshot.size(); + LOGGER.info( + DataNodePipeMessages.PIPE_START_TO_EXTRACT_HISTORICAL_TSFILE_ORIGINAL, + pipeName, + dataRegionId, + originalSequenceTsFileCount, + originalUnSequenceTsFileCount, + startIndex); + + final Map snapshotsByPartition = new TreeMap<>(); + sequenceTsFileResourceSnapshot.forEach( + resource -> + snapshotsByPartition + .computeIfAbsent( + resource.getTimePartition(), ignored -> new TsFileResourcesByPartition()) + .sequenceResources + .add(resource)); + unsequenceTsFileResourceSnapshot.forEach( + resource -> + snapshotsByPartition + .computeIfAbsent( + resource.getTimePartition(), ignored -> new TsFileResourcesByPartition()) + .unsequenceResources + .add(resource)); + + final Map resultsByPartition = new TreeMap<>(); + TsFilePartitionExtractionResult currentResult = null; try { - final int originalSequenceTsFileCount = tsFileManager.size(true); - final int originalUnSequenceTsFileCount = tsFileManager.size(false); - LOGGER.info( - DataNodePipeMessages.PIPE_START_TO_EXTRACT_HISTORICAL_TSFILE_ORIGINAL, - pipeName, - dataRegionId, - originalSequenceTsFileCount, - originalUnSequenceTsFileCount, - startIndex); - - final HistoricalTsFileExtractionStatistics statistics = - new HistoricalTsFileExtractionStatistics(); - final Map> sequenceTsFileResources2TableNames = - tsFileManager.getTsFileList(true).stream() - .peek(originalResourceList::add) - .filter(resource -> shouldExtractTsFileResource(resource, statistics)) - .collect( - Collectors.toMap( - Function.identity(), - resource -> - isModelDetected && isTableModel - ? resource.getDevices().stream() - .map(IDeviceID::getTableName) - .collect(Collectors.toSet()) - : Collections.emptySet())); - filteredTsFileResources2TableNames.putAll(sequenceTsFileResources2TableNames); - - final Map> unSequenceTsFileResources2TableNames = - tsFileManager.getTsFileList(false).stream() - .peek(originalResourceList::add) - .filter(resource -> shouldExtractTsFileResource(resource, statistics)) - .collect( - Collectors.toMap( - Function.identity(), - resource -> - isModelDetected && isTableModel - ? resource.getDevices().stream() - .map(IDeviceID::getTableName) - .collect(Collectors.toSet()) - : Collections.emptySet())); - filteredTsFileResources2TableNames.putAll(unSequenceTsFileResources2TableNames); - - filteredTsFileResources2TableNames - .keySet() - .removeIf( - resource -> { - // Pin the resource, in case the file is removed by compaction or anything. - // Will unpin it after the PipeTsFileInsertionEvent is created and pinned. - try { - if (PipeDataNodeResourceManager.tsfile() - .pinTsFileResource( - resource, shouldTransferModFile, pipeNameWithCreationTime)) { - resourcesWithPinnedModFile.add(resource); - } - return false; - } catch (final IOException e) { - ++statistics.pinFailedCount; - LOGGER.warn( - DataNodePipeMessages.PIPE_FAILED_TO_PIN_TSFILERESOURCE, - resource.getTsFilePath(), - e); - return true; - } - }); - extractedHistoricalTsFileCount = filteredTsFileResources2TableNames.size(); + for (Map.Entry entry : snapshotsByPartition.entrySet()) { + final long timePartition = entry.getKey(); + currentResult = extractTsFilesInPartition(tsFileManager, entry.getValue()); + if (!currentResult.snapshotInvalid + && !isTsFilePartitionSnapshotCurrent(tsFileManager, timePartition, entry.getValue())) { + currentResult.snapshotInvalid = true; + } - LOGGER.info( - DataNodePipeMessages.PIPE_FINISH_TO_EXTRACT_HISTORICAL_TSFILE_EXTRACTED, - pipeName, - dataRegionId, - sequenceTsFileResources2TableNames.size(), - originalSequenceTsFileCount, - unSequenceTsFileResources2TableNames.size(), - originalUnSequenceTsFileCount, - filteredTsFileResources2TableNames.size(), - originalSequenceTsFileCount + originalUnSequenceTsFileCount, - System.currentTimeMillis() - startHistoricalExtractionTime); - LOGGER.info( - DataNodePipeMessages - .MESSAGE_PIPE_ARG_ARG_HISTORICAL_TSFILE_SELECTION_SUMMARY_SELECTED_BY_PROGRESS_UNCOVERED_ARG_7B74E18D, - pipeName, - dataRegionId, - statistics.selectedByProgressUncoveredCount, - statistics.selectedByUnclosedOrClosingCount, - statistics.filteredByTimeOrPathCount, - statistics.filteredByTimeCount, - statistics.filteredByPathCount, - statistics.skippedCoveredCount, - statistics.skippedDeletedCount, - statistics.skippedGeneratedByPipeCount, - statistics.pinFailedCount); + if (currentResult.snapshotInvalid) { + final Exception unpinException = unpinTsFilesInPartition(currentResult); + if (unpinException != null) { + currentResult = null; + throw new PipeException(unpinException.getMessage(), unpinException); + } + currentResult = null; + + final TsFileResourcesByPartition refreshedResources = new TsFileResourcesByPartition(); + tsFileManager.readLock(); + try { + final Pair, List> refreshedSnapshot = + tsFileManager.getAllTsFileListForQuery( + Collections.singletonList(timePartition), null); + refreshedResources.sequenceResources.addAll(refreshedSnapshot.left); + refreshedResources.unsequenceResources.addAll(refreshedSnapshot.right); + } finally { + tsFileManager.readUnlock(); + } + + currentResult = extractTsFilesInPartition(tsFileManager, refreshedResources); + if (!currentResult.snapshotInvalid + && !isTsFilePartitionSnapshotCurrent( + tsFileManager, timePartition, refreshedResources)) { + currentResult.snapshotInvalid = true; + } + if (currentResult.snapshotInvalid) { + final Exception retryUnpinException = unpinTsFilesInPartition(currentResult); + currentResult = null; + if (retryUnpinException != null) { + throw new PipeException(retryUnpinException.getMessage(), retryUnpinException); + } + throw new PipeException( + String.format( + DataNodePipeMessages + .EXCEPTION_PIPE_ARG_ARG_TIME_PARTITION_ARG_CHANGED_DURING_HISTORICAL_TSFILE_EXTRACTION_AFTER_ONE_REFRESH_AF060BD3, + pipeName, + creationTime, + timePartition)); + } + } + resultsByPartition.put(timePartition, currentResult); + currentResult = null; + } + } catch (final RuntimeException | Error e) { + if (currentResult != null) { + final Exception unpinException = unpinTsFilesInPartition(currentResult); + if (unpinException != null) { + e.addSuppressed(unpinException); + } + } + for (TsFilePartitionExtractionResult result : resultsByPartition.values()) { + final Exception unpinException = unpinTsFilesInPartition(result); + if (unpinException != null) { + e.addSuppressed(unpinException); + } + } + throw e; + } + + final List extractedSequenceResources = new ArrayList<>(); + final List extractedUnsequenceResources = new ArrayList<>(); + final HistoricalTsFileExtractionStatistics statistics = + new HistoricalTsFileExtractionStatistics(); + int selectedSequenceTsFileCount = 0; + int selectedUnSequenceTsFileCount = 0; + for (TsFilePartitionExtractionResult result : resultsByPartition.values()) { + extractedSequenceResources.addAll(result.resources.sequenceResources); + extractedUnsequenceResources.addAll(result.resources.unsequenceResources); + filteredTsFileResources2TableNames.putAll(result.filteredResources2TableNames); + selectedSequenceTsFileCount += result.selectedSequenceTsFileCount; + selectedUnSequenceTsFileCount += result.selectedUnsequenceTsFileCount; + statistics.add(result.statistics); + } + originalResourceList.addAll(extractedSequenceResources); + originalResourceList.addAll(extractedUnsequenceResources); + extractedHistoricalTsFileCount = filteredTsFileResources2TableNames.size(); + + LOGGER.info( + DataNodePipeMessages.PIPE_FINISH_TO_EXTRACT_HISTORICAL_TSFILE_EXTRACTED, + pipeName, + dataRegionId, + selectedSequenceTsFileCount, + originalSequenceTsFileCount, + selectedUnSequenceTsFileCount, + originalUnSequenceTsFileCount, + filteredTsFileResources2TableNames.size(), + originalSequenceTsFileCount + originalUnSequenceTsFileCount, + System.currentTimeMillis() - startHistoricalExtractionTime); + LOGGER.info( + DataNodePipeMessages + .MESSAGE_PIPE_ARG_ARG_HISTORICAL_TSFILE_SELECTION_SUMMARY_SELECTED_BY_PROGRESS_UNCOVERED_ARG_7B74E18D, + pipeName, + dataRegionId, + statistics.selectedByProgressUncoveredCount, + statistics.selectedByUnclosedOrClosingCount, + statistics.filteredByTimeOrPathCount, + statistics.filteredByTimeCount, + statistics.filteredByPathCount, + statistics.skippedCoveredCount, + statistics.skippedDeletedCount, + statistics.skippedGeneratedByPipeCount, + statistics.pinFailedCount); + } + + private TsFilePartitionExtractionResult extractTsFilesInPartition( + final TsFileManager tsFileManager, final TsFileResourcesByPartition resources) { + final TsFilePartitionExtractionResult result = new TsFilePartitionExtractionResult(resources); + try { + for (TsFileResource resource : resources.sequenceResources) { + extractTsFileResource(tsFileManager, resource, true, result); + if (result.snapshotInvalid) { + break; + } + } + if (!result.snapshotInvalid) { + for (TsFileResource resource : resources.unsequenceResources) { + extractTsFileResource(tsFileManager, resource, false, result); + if (result.snapshotInvalid) { + break; + } + } + } + return result; + } catch (final RuntimeException | Error e) { + final Exception unpinException = unpinTsFilesInPartition(result); + if (unpinException != null) { + e.addSuppressed(unpinException); + } + throw e; + } + } + + private void extractTsFileResource( + final TsFileManager tsFileManager, + final TsFileResource resource, + final boolean isSequence, + final TsFilePartitionExtractionResult result) { + boolean resourceReadLockAcquired = false; + try { + resource.readLock(); + resourceReadLockAcquired = true; + tsFileManager.readLock(); + try { + if (!tsFileManager.contains(resource, isSequence)) { + result.snapshotInvalid = true; + return; + } + } finally { + tsFileManager.readUnlock(); + } + + if (!shouldExtractTsFileResource(resource, result.statistics)) { + return; + } + + final Set tableNames = + isModelDetected && isTableModel + ? resource.getDevices().stream() + .map(IDeviceID::getTableName) + .collect(Collectors.toSet()) + : Collections.emptySet(); + if (isSequence) { + ++result.selectedSequenceTsFileCount; + } else { + ++result.selectedUnsequenceTsFileCount; + } + + try { + final boolean modFilePinned = + PipeDataNodeResourceManager.tsfile() + .pinTsFileResource(resource, shouldTransferModFile, pipeNameWithCreationTime); + resourcesWithPinnedTsFile.add(resource); + if (modFilePinned) { + resourcesWithPinnedModFile.add(resource); + } + result.filteredResources2TableNames.put(resource, tableNames); + } catch (final IOException e) { + ++result.statistics.pinFailedCount; + LOGGER.warn( + DataNodePipeMessages.PIPE_FAILED_TO_PIN_TSFILERESOURCE, resource.getTsFilePath(), e); + } + } finally { + if (resourceReadLockAcquired) { + resource.readUnlock(); + } + } + } + + private boolean isTsFilePartitionSnapshotCurrent( + final TsFileManager tsFileManager, + final long timePartition, + final TsFileResourcesByPartition snapshot) { + tsFileManager.readLock(); + try { + final Pair, List> currentSnapshot = + tsFileManager.getAllTsFileListForQuery(Collections.singletonList(timePartition), null); + return areSameTsFileResources(snapshot.sequenceResources, currentSnapshot.left) + && areSameTsFileResources(snapshot.unsequenceResources, currentSnapshot.right); } finally { tsFileManager.readUnlock(); } } + private boolean areSameTsFileResources( + final List first, final List second) { + if (first.size() != second.size()) { + return false; + } + for (int i = 0; i < first.size(); ++i) { + if (first.get(i) != second.get(i)) { + return false; + } + } + return true; + } + + private Exception unpinTsFilesInPartition(final TsFilePartitionExtractionResult result) { + Exception firstException = null; + for (TsFileResource resource : new ArrayList<>(result.filteredResources2TableNames.keySet())) { + try { + unpinHistoricalTsFileResource(resource); + } catch (final IOException | RuntimeException e) { + LOGGER.warn( + DataNodePipeMessages.PIPE_FAILED_TO_UNPIN_TSFILERESOURCE_AFTER_DROPPING, + pipeName, + dataRegionId, + resource.getTsFilePath(), + e); + if (firstException == null) { + firstException = e; + } else { + firstException.addSuppressed(e); + } + } + if (!resourcesWithPinnedTsFile.contains(resource) + && !resourcesWithPinnedModFile.contains(resource)) { + result.filteredResources2TableNames.remove(resource); + } + } + return firstException; + } + + private void unpinHistoricalTsFileResource(final TsFileResource resource) throws IOException { + final PipeTsFileResourceManager manager = PipeDataNodeResourceManager.tsfile(); + Exception firstException = null; + if (resourcesWithPinnedTsFile.contains(resource)) { + try { + final File pinnedTsFile = + PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir( + resource.getTsFile(), pipeNameWithCreationTime); + manager.decreaseFileReference(pinnedTsFile, pipeNameWithCreationTime); + resourcesWithPinnedTsFile.remove(resource); + } catch (final IOException | RuntimeException e) { + firstException = e; + } + } + if (resourcesWithPinnedModFile.contains(resource)) { + try { + final File pinnedModFile = + PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir( + resource.getExclusiveModFile().getFile(), pipeNameWithCreationTime); + manager.decreaseFileReference(pinnedModFile, pipeNameWithCreationTime); + resourcesWithPinnedModFile.remove(resource); + } catch (final IOException | RuntimeException e) { + if (firstException == null) { + firstException = e; + } else { + firstException.addSuppressed(e); + } + } + } + if (firstException instanceof IOException) { + throw (IOException) firstException; + } + if (firstException != null) { + throw (RuntimeException) firstException; + } + } + private boolean shouldExtractTsFileResource( final TsFileResource resource, final HistoricalTsFileExtractionStatistics statistics) { if (!isHistoricalSourceEnabled) { @@ -1001,6 +1248,27 @@ private TimePartitionProgressIndex getTimePartitionProgressIndex( return null; } + private static class TsFileResourcesByPartition { + + private final List sequenceResources = new ArrayList<>(); + private final List unsequenceResources = new ArrayList<>(); + } + + private static class TsFilePartitionExtractionResult { + + private final TsFileResourcesByPartition resources; + private final Map> filteredResources2TableNames = new HashMap<>(); + private final HistoricalTsFileExtractionStatistics statistics = + new HistoricalTsFileExtractionStatistics(); + private int selectedSequenceTsFileCount; + private int selectedUnsequenceTsFileCount; + private boolean snapshotInvalid; + + private TsFilePartitionExtractionResult(final TsFileResourcesByPartition resources) { + this.resources = resources; + } + } + private static class HistoricalTsFileExtractionStatistics { private int selectedByProgressUncoveredCount; @@ -1012,6 +1280,18 @@ private static class HistoricalTsFileExtractionStatistics { private int skippedDeletedCount; private int skippedGeneratedByPipeCount; private int pinFailedCount; + + private void add(final HistoricalTsFileExtractionStatistics statistics) { + selectedByProgressUncoveredCount += statistics.selectedByProgressUncoveredCount; + selectedByUnclosedOrClosingCount += statistics.selectedByUnclosedOrClosingCount; + filteredByTimeOrPathCount += statistics.filteredByTimeOrPathCount; + filteredByTimeCount += statistics.filteredByTimeCount; + filteredByPathCount += statistics.filteredByPathCount; + skippedCoveredCount += statistics.skippedCoveredCount; + skippedDeletedCount += statistics.skippedDeletedCount; + skippedGeneratedByPipeCount += statistics.skippedGeneratedByPipeCount; + pinFailedCount += statistics.pinFailedCount; + } } private boolean mayTsFileResourceOverlappedWithPattern(final TsFileResource resource) { @@ -1079,11 +1359,9 @@ private boolean isTsFileResourceCoveredByTimeRange(final TsFileResource resource } private void extractDeletions( - final DeletionResourceManager deletionResourceManager, - final List resourceList) { + List allDeletionResources, final List resourceList) { LOGGER.info(DataNodePipeMessages.PIPE_START_TO_EXTRACT_DELETIONS, pipeName, dataRegionId); long startTime = System.currentTimeMillis(); - List allDeletionResources = deletionResourceManager.getAllDeletionResources(); final int originalDeletionCount = allDeletionResources.size(); // For deletions that are filtered and will not be sent, we should manually decrease its // reference count. Because the initial value of referenceCount is `ReplicaNum - 1` @@ -1238,10 +1516,8 @@ protected boolean consumeSkippedHistoricalTsFileEventIfNecessary(final TsFileRes return true; } finally { try { - PipeDataNodeResourceManager.tsfile() - .unpinTsFileResource( - resource, pipeNameWithCreationTime, resourcesWithPinnedModFile.remove(resource)); - } catch (final IOException e) { + unpinHistoricalTsFileResource(resource); + } catch (final IOException | RuntimeException e) { LOGGER.warn( DataNodePipeMessages.PIPE_FAILED_TO_UNPIN_SKIPPED_HISTORICAL_TSFILERESOURCE, pipeName, @@ -1354,10 +1630,8 @@ protected Event supplyTsFileEvent(final TsFileResource resource) { } if (shouldUnpinResource) { try { - PipeDataNodeResourceManager.tsfile() - .unpinTsFileResource( - resource, pipeNameWithCreationTime, resourcesWithPinnedModFile.remove(resource)); - } catch (final IOException e) { + unpinHistoricalTsFileResource(resource); + } catch (final IOException | RuntimeException e) { LOGGER.warn( DataNodePipeMessages.PIPE_FAILED_TO_UNPIN_TSFILERESOURCE_AFTER_CREATING, pipeName, @@ -1474,30 +1748,25 @@ public synchronized void close() { if (!isTerminateSignalSent) { PipeTerminateEvent.clearHistoricalTransferSummary(pipeName, creationTime, dataRegionId); } + final Set pinnedResources = new HashSet<>(resourcesWithPinnedTsFile); + pinnedResources.addAll(resourcesWithPinnedModFile); + for (TsFileResource resource : pinnedResources) { + try { + unpinHistoricalTsFileResource(resource); + } catch (final IOException | RuntimeException e) { + LOGGER.warn( + DataNodePipeMessages.PIPE_FAILED_TO_UNPIN_TSFILERESOURCE_AFTER_DROPPING, + pipeName, + dataRegionId, + resource.getTsFilePath(), + e); + } + } if (Objects.nonNull(pendingQueue)) { - pendingQueue.forEach( - resource -> { - if (resource instanceof TsFileResource) { - try { - PipeDataNodeResourceManager.tsfile() - .unpinTsFileResource( - (TsFileResource) resource, - pipeNameWithCreationTime, - resourcesWithPinnedModFile.remove(resource)); - } catch (final IOException e) { - LOGGER.warn( - DataNodePipeMessages.PIPE_FAILED_TO_UNPIN_TSFILERESOURCE_AFTER_DROPPING, - pipeName, - dataRegionId, - ((TsFileResource) resource).getTsFilePath()); - } - } - }); pendingQueue.clear(); pendingQueue = null; } pendingResource2ReplicateIndexForIoTV2.clear(); - resourcesWithPinnedModFile.clear(); historicalProgressReportResources.clear(); pendingHistoricalProgressIndexToReport = null; } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSourceTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSourceTest.java index 13eeafc2e906..39a6184b475b 100644 --- a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSourceTest.java +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/pipe/source/dataregion/historical/PipeHistoricalDataRegionTsFileAndDeletionSourceTest.java @@ -43,6 +43,8 @@ import org.apache.iotdb.db.pipe.event.common.tsfile.PipeTsFileInsertionEvent; import org.apache.iotdb.db.pipe.resource.PipeDataNodeResourceManager; import org.apache.iotdb.db.pipe.resource.tsfile.PipeTsFileResourceManager; +import org.apache.iotdb.db.storageengine.dataregion.modification.ModificationFile; +import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileManager; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResource; import org.apache.iotdb.db.storageengine.dataregion.tsfile.TsFileResourceStatus; import org.apache.iotdb.db.storageengine.dataregion.tsfile.timeindex.FileTimeIndex; @@ -69,6 +71,11 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; public class PipeHistoricalDataRegionTsFileAndDeletionSourceTest { @@ -102,6 +109,386 @@ public void testHistoricalTsFileReferenceFailureReleasesPinOnClose() throws Exce assertHistoricalTsFileReferenceFailure(false); } + @SuppressWarnings("unchecked") + @Test + public void testHistoricalExtractionRefreshesReplacedPartitionSnapshot() throws Exception { + final File tempDir = Files.createTempDirectory("pipeHistoricalSnapshotRefresh").toFile(); + final File partitionDir = new File(tempDir, "sequence/db/2147483645/0"); + Assert.assertTrue(partitionDir.mkdirs()); + final String pipeName = "historical_snapshot_refresh"; + final String resourcePipeName = + PipeTsFileResourceManager.getPipeTsFileResourcePipeName(pipeName, System.nanoTime()); + final PipeTsFileResourceManager pipeTsFileResourceManager = + PipeDataNodeResourceManager.tsfile(); + final TsFileManager tsFileManager = + new TsFileManager("root.db", "2147483645", tempDir.getAbsolutePath()); + final PipeHistoricalDataRegionTsFileAndDeletionSource source = + new PipeHistoricalDataRegionTsFileAndDeletionSource(); + try { + final TsFileResource staleResource = + createClosedTsFileResource(partitionDir, "1-1-0-0.tsfile", new SimpleProgressIndex(0, 1)); + final TsFileResource currentResource = + createClosedTsFileResource(partitionDir, "2-1-0-0.tsfile", new SimpleProgressIndex(0, 2)); + tsFileManager.add(staleResource, true); + final List staleSnapshot = tsFileManager.getTsFileList(true); + tsFileManager.remove(staleResource, true); + tsFileManager.add(currentResource, true); + + setPrivateField(source, "pipeName", pipeName); + setPrivateField(source, "pipeNameWithCreationTime", resourcePipeName); + setPrivateField(source, "dataRegionId", 2147483645); + setPrivateField(source, "startIndex", MinimumProgressIndex.INSTANCE); + setPrivateField(source, "isHistoricalSourceEnabled", true); + setPrivateField(source, "isDbNameCoveredByPattern", true); + + final List originalResources = new ArrayList<>(); + final Method method = + PipeHistoricalDataRegionTsFileAndDeletionSource.class.getDeclaredMethod( + "extractTsFiles", + TsFileManager.class, + List.class, + List.class, + long.class, + List.class); + method.setAccessible(true); + method.invoke( + source, + tsFileManager, + staleSnapshot, + Collections.emptyList(), + System.currentTimeMillis(), + originalResources); + + final Map> selectedResources = + (Map>) + getPrivateField(source, "filteredTsFileResources2TableNames"); + Assert.assertEquals(Collections.singletonList(currentResource), originalResources); + Assert.assertTrue(selectedResources.containsKey(currentResource)); + Assert.assertFalse(selectedResources.containsKey(staleResource)); + Assert.assertEquals(1, selectedResources.size()); + Assert.assertEquals(1, getPrivateField(source, "extractedHistoricalTsFileCount")); + } finally { + source.close(); + tsFileManager.getTsFileList(true).forEach(resource -> tsFileManager.remove(resource, true)); + pipeTsFileResourceManager.cleanPipeTsFileDir(resourcePipeName); + FileUtils.deleteFileOrDirectory(tempDir); + } + } + + @Test + public void testHistoricalExtractionRefreshesPartitionReplacedDuringScan() throws Exception { + final File tempDir = Files.createTempDirectory("pipeHistoricalConcurrentRefresh").toFile(); + final File partitionDir = new File(tempDir, "sequence/db/2147483645/0"); + Assert.assertTrue(partitionDir.mkdirs()); + final String pipeName = "historical_concurrent_refresh"; + final String resourcePipeName = + PipeTsFileResourceManager.getPipeTsFileResourcePipeName(pipeName, System.nanoTime()); + final PipeTsFileResourceManager resourceManager = PipeDataNodeResourceManager.tsfile(); + final TsFileManager tsFileManager = + new TsFileManager("root.db", "2147483645", tempDir.getAbsolutePath()); + final PipeHistoricalDataRegionTsFileAndDeletionSource source = + new PipeHistoricalDataRegionTsFileAndDeletionSource(); + final ExecutorService executor = Executors.newFixedThreadPool(2); + BlockingReadLockTsFileResource blockingResource = null; + try { + final TsFileResource firstStaleResource = + createClosedTsFileResource(partitionDir, "1-1-0-0.tsfile", new SimpleProgressIndex(0, 1)); + final File blockingFile = new File(partitionDir, "2-1-0-0.tsfile"); + Assert.assertTrue(blockingFile.createNewFile()); + final CountDownLatch readLockEntered = new CountDownLatch(1); + final CountDownLatch allowReadLock = new CountDownLatch(1); + blockingResource = + new BlockingReadLockTsFileResource(blockingFile, readLockEntered, allowReadLock); + blockingResource.setStatusForTest(TsFileResourceStatus.NORMAL); + blockingResource.updateProgressIndex(new SimpleProgressIndex(0, 2)); + final TsFileResource currentResource = + createClosedTsFileResource(partitionDir, "3-1-0-0.tsfile", new SimpleProgressIndex(0, 3)); + createExclusiveModFile(firstStaleResource); + createExclusiveModFile(currentResource); + + tsFileManager.add(firstStaleResource, true); + tsFileManager.add(blockingResource, true); + final List staleSnapshot = tsFileManager.getTsFileList(true); + setPrivateField(source, "pipeName", pipeName); + setPrivateField(source, "pipeNameWithCreationTime", resourcePipeName); + setPrivateField(source, "dataRegionId", 2147483645); + setPrivateField(source, "startIndex", MinimumProgressIndex.INSTANCE); + setPrivateField(source, "isHistoricalSourceEnabled", true); + setPrivateField(source, "isDbNameCoveredByPattern", true); + setPrivateField(source, "shouldTransferModFile", true); + + final List originalResources = new ArrayList<>(); + final Method method = + PipeHistoricalDataRegionTsFileAndDeletionSource.class.getDeclaredMethod( + "extractTsFiles", + TsFileManager.class, + List.class, + List.class, + long.class, + List.class); + method.setAccessible(true); + final BlockingReadLockTsFileResource blockedResource = blockingResource; + final Future extraction = + executor.submit( + () -> { + try { + method.invoke( + source, + tsFileManager, + staleSnapshot, + Collections.emptyList(), + System.currentTimeMillis(), + originalResources); + } catch (final ReflectiveOperationException e) { + throw new RuntimeException(e); + } + }); + + Assert.assertTrue(readLockEntered.await(10, TimeUnit.SECONDS)); + final Future replacement = + executor.submit( + () -> { + tsFileManager.remove(firstStaleResource, true); + tsFileManager.remove(blockedResource, true); + tsFileManager.add(currentResource, true); + }); + replacement.get(10, TimeUnit.SECONDS); + allowReadLock.countDown(); + extraction.get(10, TimeUnit.SECONDS); + + final File staleTsFile = + PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir( + firstStaleResource.getTsFile(), resourcePipeName); + final File staleModFile = + PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir( + firstStaleResource.getExclusiveModFile().getFile(), resourcePipeName); + final File currentTsFile = + PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir( + currentResource.getTsFile(), resourcePipeName); + final File currentModFile = + PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir( + currentResource.getExclusiveModFile().getFile(), resourcePipeName); + Assert.assertEquals(Collections.singletonList(currentResource), originalResources); + Assert.assertEquals(0, resourceManager.getFileReferenceCount(staleTsFile, resourcePipeName)); + Assert.assertEquals(0, resourceManager.getFileReferenceCount(staleModFile, resourcePipeName)); + Assert.assertEquals( + 1, resourceManager.getFileReferenceCount(currentTsFile, resourcePipeName)); + Assert.assertEquals( + 1, resourceManager.getFileReferenceCount(currentModFile, resourcePipeName)); + } finally { + if (blockingResource != null) { + blockingResource.allowReadLock(); + } + executor.shutdownNow(); + executor.awaitTermination(10, TimeUnit.SECONDS); + source.close(); + tsFileManager.getTsFileList(true).forEach(resource -> tsFileManager.remove(resource, true)); + resourceManager.cleanPipeTsFileDir(resourcePipeName); + FileUtils.deleteFileOrDirectory(tempDir); + } + } + + @Test + public void testHistoricalModPinFailureRollsBackTsFilePin() throws Exception { + final File tempDir = Files.createTempDirectory("pipeHistoricalModPinFailure").toFile(); + final File partitionDir = new File(tempDir, "sequence/db/2147483645/0"); + Assert.assertTrue(partitionDir.mkdirs()); + final String pipeName = "historical_mod_pin_failure"; + final String resourcePipeName = + PipeTsFileResourceManager.getPipeTsFileResourcePipeName(pipeName, System.nanoTime()); + final PipeTsFileResourceManager resourceManager = PipeDataNodeResourceManager.tsfile(); + final TsFileManager tsFileManager = + new TsFileManager("root.db", "2147483645", tempDir.getAbsolutePath()); + final PipeHistoricalDataRegionTsFileAndDeletionSource source = + new PipeHistoricalDataRegionTsFileAndDeletionSource(); + final AtomicBoolean failModPathResolution = new AtomicBoolean(true); + try { + final TsFileResource resource = + createClosedTsFileResource(partitionDir, "1-1-0-0.tsfile", new SimpleProgressIndex(0, 1)); + final File modFile = resource.getExclusiveModFile().getFile(); + Files.write(modFile.toPath(), new byte[] {1}); + resource.setExclusiveModFile( + new ModificationFile( + new ModPathFailingFile(modFile.getPath(), failModPathResolution), false)); + tsFileManager.add(resource, true); + + setPrivateField(source, "pipeName", pipeName); + setPrivateField(source, "pipeNameWithCreationTime", resourcePipeName); + setPrivateField(source, "dataRegionId", 2147483645); + setPrivateField(source, "startIndex", MinimumProgressIndex.INSTANCE); + setPrivateField(source, "isHistoricalSourceEnabled", true); + setPrivateField(source, "isDbNameCoveredByPattern", true); + setPrivateField(source, "shouldTransferModFile", true); + final Method method = + PipeHistoricalDataRegionTsFileAndDeletionSource.class.getDeclaredMethod( + "extractTsFiles", + TsFileManager.class, + List.class, + List.class, + long.class, + List.class); + method.setAccessible(true); + method.invoke( + source, + tsFileManager, + tsFileManager.getTsFileList(true), + Collections.emptyList(), + System.currentTimeMillis(), + new ArrayList()); + + final File pinnedTsFile = + PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir( + resource.getTsFile(), resourcePipeName); + Assert.assertEquals(0, resourceManager.getFileReferenceCount(pinnedTsFile, resourcePipeName)); + Assert.assertTrue( + ((Set) getPrivateField(source, "resourcesWithPinnedTsFile")).isEmpty()); + Assert.assertTrue( + ((Set) getPrivateField(source, "resourcesWithPinnedModFile")).isEmpty()); + } finally { + source.close(); + tsFileManager.getTsFileList(true).forEach(resource -> tsFileManager.remove(resource, true)); + resourceManager.cleanPipeTsFileDir(resourcePipeName); + FileUtils.deleteFileOrDirectory(tempDir); + } + } + + @SuppressWarnings("unchecked") + @Test + public void testCloseRetriesTsFileUnpinAfterTsFileUnpinFailed() throws Exception { + final File tempDir = Files.createTempDirectory("pipeHistoricalTsFileUnpinRetry").toFile(); + final File partitionDir = new File(tempDir, "sequence/db/2147483645/0"); + Assert.assertTrue(partitionDir.mkdirs()); + final String pipeName = "historical_tsfile_unpin_retry"; + final String resourcePipeName = + PipeTsFileResourceManager.getPipeTsFileResourcePipeName(pipeName, System.nanoTime()); + final PipeTsFileResourceManager resourceManager = PipeDataNodeResourceManager.tsfile(); + final PipeHistoricalDataRegionTsFileAndDeletionSource source = + new PipeHistoricalDataRegionTsFileAndDeletionSource(); + final AtomicBoolean failTsFilePathResolution = new AtomicBoolean(); + try { + final TsFileResource resource = + createClosedTsFileResource(partitionDir, "1-1-0-0.tsfile", new SimpleProgressIndex(0, 1)); + resource.setFile( + new CanonicalPathFailingFile(resource.getTsFilePath(), failTsFilePathResolution)); + createExclusiveModFile(resource); + Assert.assertTrue(resourceManager.pinTsFileResource(resource, true, resourcePipeName)); + final File pinnedTsFile = + PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir( + resource.getTsFile(), resourcePipeName); + final File pinnedModFile = + PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir( + resource.getExclusiveModFile().getFile(), resourcePipeName); + ((Set) getPrivateField(source, "resourcesWithPinnedTsFile")).add(resource); + ((Set) getPrivateField(source, "resourcesWithPinnedModFile")).add(resource); + setPrivateField(source, "pipeName", pipeName); + setPrivateField(source, "pipeNameWithCreationTime", resourcePipeName); + setPrivateField(source, "dataRegionId", 2147483645); + + failTsFilePathResolution.set(true); + final Method unpinMethod = + PipeHistoricalDataRegionTsFileAndDeletionSource.class.getDeclaredMethod( + "unpinHistoricalTsFileResource", TsFileResource.class); + unpinMethod.setAccessible(true); + try { + unpinMethod.invoke(source, resource); + Assert.fail("Expected the TsFile path resolution to fail"); + } catch (final java.lang.reflect.InvocationTargetException e) { + Assert.assertTrue(e.getCause() instanceof IOException); + } + + Assert.assertEquals(1, resourceManager.getFileReferenceCount(pinnedTsFile, resourcePipeName)); + Assert.assertEquals( + 0, resourceManager.getFileReferenceCount(pinnedModFile, resourcePipeName)); + Assert.assertTrue( + ((Set) getPrivateField(source, "resourcesWithPinnedTsFile")) + .contains(resource)); + Assert.assertFalse( + ((Set) getPrivateField(source, "resourcesWithPinnedModFile")) + .contains(resource)); + + failTsFilePathResolution.set(false); + source.close(); + Assert.assertEquals(0, resourceManager.getFileReferenceCount(pinnedTsFile, resourcePipeName)); + Assert.assertTrue( + ((Set) getPrivateField(source, "resourcesWithPinnedTsFile")).isEmpty()); + } finally { + failTsFilePathResolution.set(false); + source.close(); + resourceManager.cleanPipeTsFileDir(resourcePipeName); + FileUtils.deleteFileOrDirectory(tempDir); + } + } + + @SuppressWarnings("unchecked") + @Test + public void testCloseRetriesModUnpinAfterTsFileUnpinSucceeded() throws Exception { + final File tempDir = Files.createTempDirectory("pipeHistoricalModUnpinRetry").toFile(); + final File partitionDir = new File(tempDir, "sequence/db/2147483645/0"); + Assert.assertTrue(partitionDir.mkdirs()); + final String pipeName = "historical_mod_unpin_retry"; + final String resourcePipeName = + PipeTsFileResourceManager.getPipeTsFileResourcePipeName(pipeName, System.nanoTime()); + final PipeTsFileResourceManager resourceManager = PipeDataNodeResourceManager.tsfile(); + final PipeHistoricalDataRegionTsFileAndDeletionSource source = + new PipeHistoricalDataRegionTsFileAndDeletionSource(); + final AtomicBoolean failModPathResolution = new AtomicBoolean(); + try { + final TsFileResource resource = + createClosedTsFileResource(partitionDir, "1-1-0-0.tsfile", new SimpleProgressIndex(0, 1)); + final File modFile = resource.getExclusiveModFile().getFile(); + Files.write(modFile.toPath(), new byte[] {1}); + resource.setExclusiveModFile( + new ModificationFile( + new ModPathFailingFile(modFile.getPath(), failModPathResolution), false)); + Assert.assertTrue(resourceManager.pinTsFileResource(resource, true, resourcePipeName)); + final File pinnedTsFile = + PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir( + resource.getTsFile(), resourcePipeName); + final File pinnedModFile = + PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir( + resource.getExclusiveModFile().getFile(), resourcePipeName); + ((Set) getPrivateField(source, "resourcesWithPinnedTsFile")).add(resource); + ((Set) getPrivateField(source, "resourcesWithPinnedModFile")).add(resource); + setPrivateField(source, "pipeName", pipeName); + setPrivateField(source, "pipeNameWithCreationTime", resourcePipeName); + setPrivateField(source, "dataRegionId", 2147483645); + + failModPathResolution.set(true); + final Method unpinMethod = + PipeHistoricalDataRegionTsFileAndDeletionSource.class.getDeclaredMethod( + "unpinHistoricalTsFileResource", TsFileResource.class); + unpinMethod.setAccessible(true); + try { + unpinMethod.invoke(source, resource); + Assert.fail("Expected the mod path resolution to fail"); + } catch (final java.lang.reflect.InvocationTargetException e) { + Assert.assertTrue(e.getCause() instanceof IOException); + } + + Assert.assertEquals(0, resourceManager.getFileReferenceCount(pinnedTsFile, resourcePipeName)); + Assert.assertEquals( + 1, resourceManager.getFileReferenceCount(pinnedModFile, resourcePipeName)); + Assert.assertFalse( + ((Set) getPrivateField(source, "resourcesWithPinnedTsFile")) + .contains(resource)); + Assert.assertTrue( + ((Set) getPrivateField(source, "resourcesWithPinnedModFile")) + .contains(resource)); + + failModPathResolution.set(false); + source.close(); + Assert.assertEquals( + 0, resourceManager.getFileReferenceCount(pinnedModFile, resourcePipeName)); + Assert.assertTrue( + ((Set) getPrivateField(source, "resourcesWithPinnedModFile")).isEmpty()); + } finally { + failModPathResolution.set(false); + source.close(); + resourceManager.cleanPipeTsFileDir(resourcePipeName); + FileUtils.deleteFileOrDirectory(tempDir); + } + } + @SuppressWarnings("unchecked") private void assertHistoricalTsFileReferenceFailure(final boolean retry) throws Exception { final File tempDir = Files.createTempDirectory("pipeHistoricalReferenceFailure").toFile(); @@ -145,6 +532,7 @@ private void assertHistoricalTsFileReferenceFailure(final boolean retry) throws new ArrayDeque(Collections.singletonList(resource))); manager.pinTsFileResource(resource, false, resourcePipeName); + ((Set) getPrivateField(source, "resourcesWithPinnedTsFile")).add(resource); final File pinnedFile = PipeTsFileResourceManager.getHardlinkOrCopiedFileInPipeDir( resource.getTsFile(), resourcePipeName); @@ -230,6 +618,7 @@ private void assertSuppliedTsFileTimeFiltering( setPrivateField(source, "historicalDataExtractionStartTime", -10L); setPrivateField(source, "historicalDataExtractionEndTime", 10L); setPrivateField(source, "sloppyTimeRange", looseTimeRange); + ((Set) getPrivateField(source, "resourcesWithPinnedTsFile")).add(resource); ((Map>) getPrivateField(source, "filteredTsFileResources2TableNames")) .put(resource, Collections.emptySet()); @@ -975,6 +1364,12 @@ private static TsFileResource createTsFileResource(final File tempDir, final Str return new TsFileResource(file); } + private static void createExclusiveModFile(final TsFileResource resource) throws IOException { + final File modFile = resource.getExclusiveModFile().getFile(); + Files.write(modFile.toPath(), new byte[] {1}); + resource.setExclusiveModFile(new ModificationFile(modFile, false)); + } + private static TsFileResource createTsFileResource( final File tempDir, final long timePartitionId, final String fileName) throws IOException { final File regionDir = new File(tempDir, "1"); @@ -1133,6 +1528,58 @@ public String getCanonicalPath() throws IOException { } } + private static class ModPathFailingFile extends File { + + private final AtomicBoolean failPathResolution; + + private ModPathFailingFile(final String path, final AtomicBoolean failPathResolution) { + super(path); + this.failPathResolution = failPathResolution; + } + + @Override + public String getCanonicalPath() throws IOException { + if (failPathResolution.get()) { + throw new IOException("mock mod path resolution failure"); + } + return super.getCanonicalPath(); + } + } + + private static class BlockingReadLockTsFileResource extends TsFileResource { + + private final CountDownLatch readLockEntered; + private final CountDownLatch allowReadLock; + private final AtomicBoolean shouldBlock = new AtomicBoolean(true); + + private BlockingReadLockTsFileResource( + final File file, final CountDownLatch readLockEntered, final CountDownLatch allowReadLock) { + super(file); + this.readLockEntered = readLockEntered; + this.allowReadLock = allowReadLock; + } + + @Override + public void readLock() { + if (shouldBlock.compareAndSet(true, false)) { + readLockEntered.countDown(); + try { + if (!allowReadLock.await(10, TimeUnit.SECONDS)) { + throw new IllegalStateException("Timed out waiting to acquire the test read lock"); + } + } catch (final InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } + super.readLock(); + } + + private void allowReadLock() { + allowReadLock.countDown(); + } + } + private static class TestablePipeHistoricalDataRegionTsFileAndDeletionSource extends PipeHistoricalDataRegionTsFileAndDeletionSource {