Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -651,6 +651,9 @@ public final class DataNodePipeMessages {
"Pipe {}@{}: failed to unpin TsFileResource after creating event, original path: {}";
public static final String PIPE_FAILED_TO_UNPIN_TSFILERESOURCE_AFTER_DROPPING =
"Pipe {}@{}: failed to unpin TsFileResource after dropping pipe, original path: {}";
public static final String
EXCEPTION_PIPE_ARG_ARG_TIME_PARTITION_ARG_CHANGED_DURING_HISTORICAL_TSFILE_EXTRACTION_AFTER_ONE_REFRESH_AF060BD3 =
"Pipe {}@{}: time partition {} changed during historical TsFile extraction after one refresh";
public static final String PIPE_FINISH_TO_EXTRACT_DELETIONS_EXTRACT_DELETIONS =
"Pipe {}@{}: finish to extract deletions, extract deletions count {}/{}, took {} ms";
public static final String PIPE_FINISH_TO_EXTRACT_HISTORICAL_TSFILE_EXTRACTED =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -617,6 +617,9 @@ public final class DataNodePipeMessages {
"Pipe {}@{}:unpin TsFileResource after creating event, original path: {} 失败";
public static final String PIPE_FAILED_TO_UNPIN_TSFILERESOURCE_AFTER_DROPPING =
"Pipe {}@{}:unpin TsFileResource after dropping pipe, original path: {} 失败";
public static final String
EXCEPTION_PIPE_ARG_ARG_TIME_PARTITION_ARG_CHANGED_DURING_HISTORICAL_TSFILE_EXTRACTION_AFTER_ONE_REFRESH_AF060BD3 =
"Pipe {}@{}:历史 TsFile 提取期间,时间分区 {} 在刷新一次后仍发生变化";
public static final String PIPE_FINISH_TO_EXTRACT_DELETIONS_EXTRACT_DELETIONS =
"Pipe {}@{}:finish to extract deletions, extract deletions count {}/{}, took {} ms";
public static final String PIPE_FINISH_TO_EXTRACT_HISTORICAL_TSFILE_EXTRACTED =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import org.apache.iotdb.db.protocol.session.InternalClientSession;
import org.apache.iotdb.db.protocol.session.SessionManager;
import org.apache.iotdb.db.storageengine.StorageEngine;
import org.apache.iotdb.db.storageengine.dataregion.DataRegion;
import org.apache.iotdb.pipe.api.annotation.TableModel;
import org.apache.iotdb.pipe.api.annotation.TreeModel;
import org.apache.iotdb.pipe.api.customizer.configuration.PipeExtractorRuntimeConfiguration;
Expand Down Expand Up @@ -552,20 +553,12 @@ public void start() throws Exception {
.runIfPresent(
dataRegionIdObject,
(dataRegion -> {
dataRegion.writeLock(
String.format(
DataNodePipeMessages.MESSAGE_PIPE_STARTING_ARG_A09517B7,
IoTDBDataRegionSource.class.getName()));
try {
startHistoricalExtractorAndRealtimeExtractor(exceptionHolder);
} finally {
dataRegion.writeUnlock();
}
startHistoricalExtractorAndRealtimeExtractor(exceptionHolder, dataRegion);
}))
|| StorageEngine.getInstance()
.runIfAbsent(
dataRegionIdObject,
() -> startHistoricalExtractorAndRealtimeExtractor(exceptionHolder))) {
() -> startHistoricalExtractorAndRealtimeExtractor(exceptionHolder, null))) {
rethrowExceptionIfAny(exceptionHolder);
completionSourceStarted.set(true);

Expand All @@ -583,14 +576,26 @@ public void start() throws Exception {
}

private void startHistoricalExtractorAndRealtimeExtractor(
final AtomicReference<Exception> exceptionHolder) {
final AtomicReference<Exception> exceptionHolder, final DataRegion dataRegion) {
try {
// Start realtimeSource first to avoid losing data. This may cause some
// retransmission, yet it is OK according to the idempotency of IoTDB.
// Note: The order of historical collection is flushing data -> adding all tsFile events.
// There can still be writing when tsFile events are added. If we start
// realtimeSource after the process, then this part of data will be lost.
realtimeSource.start();
if (Objects.nonNull(dataRegion)) {
dataRegion.writeLock(
String.format(
DataNodePipeMessages.MESSAGE_PIPE_STARTING_ARG_A09517B7,
IoTDBDataRegionSource.class.getName()));
try {
realtimeSource.start();
} finally {
dataRegion.writeUnlock();
}
} else {
realtimeSource.start();
}
historicalSource.start();
} catch (final Exception e) {
exceptionHolder.set(e);
Expand Down
Loading
Loading