From 351a7a500fb303e61b4bb08bfbee46b8fbfc00a0 Mon Sep 17 00:00:00 2001 From: Caideyipi <87789683+Caideyipi@users.noreply.github.com> Date: Fri, 20 Nov 2026 17:50:26 +0800 Subject: [PATCH] Stabilize IoTConsensus peer recovery notifications --- .../iotdb/consensus/iot/IoTConsensus.java | 104 +++++++++----- .../consensus/iot/IoTConsensusServerImpl.java | 15 +- .../iot/logdispatcher/LogDispatcher.java | 15 +- .../iot/IoTConsensusRecoveryTest.java | 136 ++++++++++++++++++ 4 files changed, 236 insertions(+), 34 deletions(-) diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java index a52f24051472..5a0d75659562 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensus.java @@ -88,6 +88,7 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; import java.util.function.Consumer; import java.util.stream.Collectors; @@ -177,25 +178,47 @@ public IoTConsensus(ConsensusConfig config, Registry registry) { @Override public synchronized void start() throws IOException { - initAndRecover(); - service.initSyncedServiceImpl(new IoTConsensusRPCServiceProcessor(this)); try { - registerManager.register(service); - } catch (StartupException e) { - throw new IOException(e); + initAndRecover(); + service.initSyncedServiceImpl(new IoTConsensusRPCServiceProcessor(this)); + try { + registerManager.register(service); + } catch (StartupException e) { + throw new IOException(e); + } + if (updateReaderFuture == null) { + updateReaderFuture = + ScheduledExecutorUtil.safelyScheduleWithFixedDelay( + backgroundTaskService, + () -> + stateMachineMap + .values() + .forEach(impl -> impl.getLogDispatcher().checkAndFlushIndex()), + READER_UPDATE_INTERVAL_IN_MINUTES, + READER_UPDATE_INTERVAL_IN_MINUTES, + TimeUnit.MINUTES); + } + } catch (IOException | RuntimeException e) { + cleanupAfterFailedStart(e); + throw e; + } + } + + private void cleanupAfterFailedStart(final Throwable startupFailure) { + if (updateReaderFuture != null) { + updateReaderFuture.cancel(false); + updateReaderFuture = null; } - if (updateReaderFuture == null) { - updateReaderFuture = - ScheduledExecutorUtil.safelyScheduleWithFixedDelay( - backgroundTaskService, - () -> - stateMachineMap - .values() - .forEach(impl -> impl.getLogDispatcher().checkAndFlushIndex()), - READER_UPDATE_INTERVAL_IN_MINUTES, - READER_UPDATE_INTERVAL_IN_MINUTES, - TimeUnit.MINUTES); + registerManager.deregisterAll(); + for (IoTConsensusServerImpl impl : stateMachineMap.values()) { + try { + impl.stop(); + } catch (RuntimeException e) { + startupFailure.addSuppressed(e); + } } + stateMachineMap.clear(); + pendingPeerCreatedCallbacks.clear(); } private void initAndRecover() throws IOException { @@ -233,6 +256,7 @@ private void initAndRecover() throws IOException { } } if (correctPeerListBeforeStart != null) { + final AtomicReference peerListResetFailure = new AtomicReference<>(); BiConsumer> resetPeerListWithoutThrow = (consensusGroupId, peers) -> { try { @@ -240,6 +264,7 @@ private void initAndRecover() throws IOException { } catch (ConsensusGroupNotExistException ignore) { } catch (Exception e) { + peerListResetFailure.compareAndSet(null, e); logger.warn(ConsensusMessages.FAILED_TO_RESET_PEER_LIST_WHILE_START, e); } }; @@ -253,23 +278,38 @@ private void initAndRecover() throws IOException { .forEach( consensusGroupId -> resetPeerListWithoutThrow.accept(consensusGroupId, Collections.emptyList())); + final Exception resetFailure = peerListResetFailure.get(); + if (resetFailure != null) { + throw new IOException( + ConsensusMessages.FAILED_TO_RESET_PEER_LIST_WHILE_START, resetFailure); + } + } + for (Map.Entry entry : stateMachineMap.entrySet()) { + final BiConsumer callback = onPeerRecovering; + if (callback != null) { + try { + callback.accept(entry.getKey(), entry.getValue()); + } catch (RuntimeException e) { + throw new IOException(e); + } + } + } + if (correctPeerListBeforeStart != null) { + stateMachineMap.values().forEach(IoTConsensusServerImpl::markPeerConfigurationRecovered); + } + for (Map.Entry entry : stateMachineMap.entrySet()) { + try { + entry.getValue().start(); + // Subscription metadata may have been loaded before any local consensus peers existed. + // Recovered peers must trigger the same binding as newly created peers, after startup has + // restored their indices and installed the pre-start WAL-retention guards. + notifyPeerCreated(entry.getKey(), entry.getValue()); + } catch (RuntimeException e) { + throw new IOException(e); + } finally { + pendingPeerCreatedCallbacks.remove(entry.getKey()); + } } - stateMachineMap.forEach( - (consensusGroupId, consensus) -> { - try { - final BiConsumer callback = onPeerRecovering; - if (callback != null) { - callback.accept(consensusGroupId, consensus); - } - consensus.start(); - // Subscription metadata may have been loaded before any local consensus peers existed. - // Recovered peers must trigger the same binding as newly created peers, after startup - // has restored their indices and installed the pre-start WAL-retention guards. - notifyPeerCreated(consensusGroupId, consensus); - } finally { - pendingPeerCreatedCallbacks.remove(consensusGroupId); - } - }); } @Override diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java index dda7c2b02294..9951c8b52dc1 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/IoTConsensusServerImpl.java @@ -152,6 +152,7 @@ public class IoTConsensusServerImpl { private final Set configuration = ConcurrentHashMap.newKeySet(); private final AtomicLong searchIndex; private final Object retentionPublicationLock = new Object(); + private boolean recoveringPeerConfiguration; private final LogDispatcher logDispatcher; private IoTConsensusConfig config; private final ConsensusReqReader consensusReqReader; @@ -249,6 +250,7 @@ public IoTConsensusServerImpl( this.cacheQueueMap = new ConcurrentHashMap<>(); this.syncClientManager = syncClientManager; this.configuration.addAll(configuration); + this.recoveringPeerConfiguration = configuration.isEmpty(); this.backgroundTaskService = backgroundTaskService; this.config = config; this.userDataTransferAuditHandler = userDataTransferAuditHandler; @@ -271,12 +273,20 @@ public IStateMachine getStateMachine() { } public void start() { - checkAndUpdateIndex(); + synchronized (retentionPublicationLock) { + checkAndUpdateIndex(); + } MetricService.getInstance().addMetricSet(this.ioTConsensusServerMetrics); stateMachine.start(); logDispatcher.start(); } + void markPeerConfigurationRecovered() { + synchronized (retentionPublicationLock) { + recoveringPeerConfiguration = false; + } + } + public void stop() { logDispatcher.stop(); persistLatestWriterMeta(true); @@ -1486,6 +1496,9 @@ void checkAndUpdateIndex() { */ public void checkAndUpdateSafeDeletedSearchIndex() { synchronized (retentionPublicationLock) { + if (recoveringPeerConfiguration) { + return; + } if (configuration.isEmpty()) { logger.error(IoTConsensusMessages.CONFIGURATION_EMPTY_UNEXPECTED); return; diff --git a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java index 7cb5e7a63e03..593d77322f98 100644 --- a/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java +++ b/iotdb-core/consensus/src/main/java/org/apache/iotdb/consensus/iot/logdispatcher/LogDispatcher.java @@ -105,7 +105,11 @@ private void initLogSyncThreadPool() { public synchronized void start() { if (!threads.isEmpty()) { - threads.forEach(logDispatcherThread -> executorService.submit(logDispatcherThread)); + threads.forEach( + logDispatcherThread -> { + logDispatcherThread.markSubmitted(); + executorService.submit(logDispatcherThread); + }); } } @@ -140,6 +144,7 @@ public synchronized void addLogDispatcherThread( initLogSyncThreadPool(); } if (startNow) { + thread.markSubmitted(); executorService.submit(thread); } } @@ -309,6 +314,7 @@ public class LogDispatcherThread implements Runnable { private final LogDispatcherThreadMetrics logDispatcherThreadMetrics; private final CountDownLatch runFinished = new CountDownLatch(1); + private volatile boolean submitted; private volatile long lastIdleWriterSafeTimeBarrierSentTimeMs = 0L; public LogDispatcherThread(Peer peer, IoTConsensusConfig config, long initialSyncIndex) { @@ -385,8 +391,15 @@ public void stop() { processStopped(); } + private void markSubmitted() { + submitted = true; + } + private void setStopped() { stopped = true; + if (!submitted) { + runFinished.countDown(); + } } private void processStopped() { diff --git a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/IoTConsensusRecoveryTest.java b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/IoTConsensusRecoveryTest.java index 4e7131f1a5c0..f651985b8869 100644 --- a/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/IoTConsensusRecoveryTest.java +++ b/iotdb-core/consensus/src/test/java/org/apache/iotdb/consensus/iot/IoTConsensusRecoveryTest.java @@ -23,8 +23,12 @@ import org.apache.iotdb.common.rpc.thrift.TEndPoint; import org.apache.iotdb.commons.consensus.ConsensusGroupId; import org.apache.iotdb.commons.consensus.DataRegionId; +import org.apache.iotdb.commons.request.IConsensusRequest; +import org.apache.iotdb.consensus.common.DataSet; import org.apache.iotdb.consensus.common.Peer; import org.apache.iotdb.consensus.config.ConsensusConfig; +import org.apache.iotdb.consensus.iot.log.ConsensusReqReader; +import org.apache.iotdb.consensus.iot.util.FakeConsensusReqReader; import org.apache.iotdb.consensus.iot.util.TestStateMachine; import org.junit.After; @@ -33,6 +37,7 @@ import org.junit.Test; import org.junit.rules.TemporaryFolder; +import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -44,7 +49,13 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.anyLong; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; public class IoTConsensusRecoveryTest { @@ -122,6 +133,119 @@ public void testRecoveredPeersNotifyAfterStarting() throws Exception { assertFalse(consensus.isPeerCreationCallbackPending(secondRegion)); } + @Test + public void testRecoveryPublishesWalRetentionAfterRestoringPeers() throws Exception { + final DataRegionId regionId = new DataRegionId(1); + final List peers = + Arrays.asList( + new Peer(regionId, 2, new TEndPoint("127.0.0.1", 1)), + new Peer(regionId, config.getThisNodeId(), config.getThisNodeEndPoint())); + consensus.createLocalPeer(regionId, peers); + consensus.stop(); + + final FakeConsensusReqReader reader = mock(FakeConsensusReqReader.class); + when(reader.getReqIterator(anyLong())).thenReturn(mock(ConsensusReqReader.ReqIterator.class)); + consensus = newConsensus(reader); + consensus.recordCorrectPeerListBeforeStarting(Collections.singletonMap(regionId, peers)); + IoTConsensus.onPeerRecovering = + (groupId, server) -> { + server.registerDetachedSubscriptionRetention( + "test-retention", + new SubscriptionWalRetentionPolicy( + "test-topic", + SubscriptionWalRetentionPolicy.UNBOUNDED, + SubscriptionWalRetentionPolicy.UNBOUNDED), + () -> 0L); + verify(reader, never()).setSafelyDeletedSearchIndex(anyLong()); + verify(reader, never()).setSubscriptionRetainedMinVersionId(anyLong()); + }; + consensus.start(); + + verify(reader, never()).setSafelyDeletedSearchIndex(Long.MAX_VALUE); + verify(reader).setSafelyDeletedSearchIndex(Long.MIN_VALUE + 1); + verify(reader).setSubscriptionRetainedMinVersionId(0L); + } + + @Test + public void testSingleReplicaRecoveryPublishesUnboundedRetentionAfterPeerRecovery() + throws Exception { + final DataRegionId regionId = new DataRegionId(1); + final List peers = + Collections.singletonList( + new Peer(regionId, config.getThisNodeId(), config.getThisNodeEndPoint())); + consensus.createLocalPeer(regionId, peers); + consensus.stop(); + + final FakeConsensusReqReader reader = mock(FakeConsensusReqReader.class); + consensus = newConsensus(reader); + consensus.recordCorrectPeerListBeforeStarting(Collections.singletonMap(regionId, peers)); + IoTConsensus.onPeerRecovering = + (groupId, server) -> { + verify(reader, never()).setSafelyDeletedSearchIndex(anyLong()); + verify(reader, never()).setSubscriptionRetainedMinVersionId(anyLong()); + }; + + consensus.start(); + + verify(reader).setSafelyDeletedSearchIndex(Long.MAX_VALUE); + verify(reader).setSubscriptionRetainedMinVersionId(Long.MAX_VALUE); + } + + @Test + public void testRecoveryDoesNotPublishWalRetentionWhenPeerRestoreFails() throws Exception { + final DataRegionId regionId = new DataRegionId(1); + final List peers = + Arrays.asList( + new Peer(regionId, 2, new TEndPoint(config.getThisNodeEndPoint().getIp(), 1)), + new Peer(regionId, config.getThisNodeId(), config.getThisNodeEndPoint())); + consensus.createLocalPeer(regionId, peers); + consensus.stop(); + + final FakeConsensusReqReader reader = mock(FakeConsensusReqReader.class); + when(reader.getReqIterator(anyLong())).thenReturn(mock(ConsensusReqReader.ReqIterator.class)); + consensus = newConsensus(reader); + consensus.recordCorrectPeerListBeforeStarting(Collections.singletonMap(regionId, peers)); + final IllegalStateException recoveryFailure = new IllegalStateException(); + IoTConsensus.onPeerRecovering = + (groupId, server) -> { + throw recoveryFailure; + }; + + final IOException exception = assertThrows(IOException.class, () -> consensus.start()); + assertSame(recoveryFailure, exception.getCause()); + verify(reader, never()).setSafelyDeletedSearchIndex(anyLong()); + verify(reader, never()).setSubscriptionRetainedMinVersionId(anyLong()); + assertTrue(consensus.getAllConsensusGroupIds().isEmpty()); + assertFalse(consensus.isPeerCreationCallbackPending(regionId)); + + IoTConsensus.onPeerRecovering = null; + consensus.start(); + + verify(reader).setSafelyDeletedSearchIndex(0L); + } + + @Test + public void testPeerListResetFailureIsReportedAndCleanedUp() throws Exception { + final DataRegionId regionId = new DataRegionId(1); + final List peers = + Arrays.asList( + new Peer(regionId, 2, new TEndPoint(config.getThisNodeEndPoint().getIp(), 1)), + new Peer(regionId, config.getThisNodeId(), config.getThisNodeEndPoint())); + consensus.createLocalPeer(regionId, peers); + consensus.stop(); + + final FakeConsensusReqReader reader = mock(FakeConsensusReqReader.class); + when(reader.getReqIterator(anyLong())).thenThrow(new IllegalStateException()); + consensus = newConsensus(reader); + consensus.recordCorrectPeerListBeforeStarting(Collections.singletonMap(regionId, peers)); + + assertThrows(IOException.class, () -> consensus.start()); + + verify(reader, never()).setSafelyDeletedSearchIndex(anyLong()); + verify(reader, never()).setSubscriptionRetainedMinVersionId(anyLong()); + assertTrue(consensus.getAllConsensusGroupIds().isEmpty()); + } + @Test public void testRecoveryContinuesAfterPeerNotificationFailure() throws Exception { final DataRegionId firstRegion = new DataRegionId(1); @@ -178,6 +302,18 @@ private IoTConsensus newConsensus() { return instance; } + private IoTConsensus newConsensus(final FakeConsensusReqReader reader) { + return new IoTConsensus( + config, + groupId -> + new TestStateMachine() { + @Override + public synchronized DataSet read(final IConsensusRequest request) { + return reader; + } + }); + } + private void createPeer(final DataRegionId regionId) throws Exception { final List peers = Collections.singletonList(