Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiConsumer;
import java.util.function.Consumer;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -177,25 +178,47 @@ public IoTConsensus(ConsensusConfig config, Registry registry) {

@Override
public synchronized void start() throws IOException {
initAndRecover();
service.initSyncedServiceImpl(new IoTConsensusRPCServiceProcessor(this));
try {
registerManager.register(service);
} catch (StartupException e) {
throw new IOException(e);
initAndRecover();
service.initSyncedServiceImpl(new IoTConsensusRPCServiceProcessor(this));
try {
registerManager.register(service);
} catch (StartupException e) {
throw new IOException(e);
}
if (updateReaderFuture == null) {
updateReaderFuture =
ScheduledExecutorUtil.safelyScheduleWithFixedDelay(
backgroundTaskService,
() ->
stateMachineMap
.values()
.forEach(impl -> impl.getLogDispatcher().checkAndFlushIndex()),
READER_UPDATE_INTERVAL_IN_MINUTES,
READER_UPDATE_INTERVAL_IN_MINUTES,
TimeUnit.MINUTES);
}
} catch (IOException | RuntimeException e) {
cleanupAfterFailedStart(e);
throw e;
}
}

private void cleanupAfterFailedStart(final Throwable startupFailure) {
if (updateReaderFuture != null) {
updateReaderFuture.cancel(false);
updateReaderFuture = null;
}
if (updateReaderFuture == null) {
updateReaderFuture =
ScheduledExecutorUtil.safelyScheduleWithFixedDelay(
backgroundTaskService,
() ->
stateMachineMap
.values()
.forEach(impl -> impl.getLogDispatcher().checkAndFlushIndex()),
READER_UPDATE_INTERVAL_IN_MINUTES,
READER_UPDATE_INTERVAL_IN_MINUTES,
TimeUnit.MINUTES);
registerManager.deregisterAll();
for (IoTConsensusServerImpl impl : stateMachineMap.values()) {
try {
impl.stop();
} catch (RuntimeException e) {
startupFailure.addSuppressed(e);
}
}
stateMachineMap.clear();
pendingPeerCreatedCallbacks.clear();
}

private void initAndRecover() throws IOException {
Expand Down Expand Up @@ -233,13 +256,15 @@ private void initAndRecover() throws IOException {
}
}
if (correctPeerListBeforeStart != null) {
final AtomicReference<Exception> peerListResetFailure = new AtomicReference<>();
BiConsumer<ConsensusGroupId, List<Peer>> resetPeerListWithoutThrow =
(consensusGroupId, peers) -> {
try {
resetPeerListImpl(consensusGroupId, peers, false);
} catch (ConsensusGroupNotExistException ignore) {

} catch (Exception e) {
peerListResetFailure.compareAndSet(null, e);
logger.warn(ConsensusMessages.FAILED_TO_RESET_PEER_LIST_WHILE_START, e);
}
};
Expand All @@ -253,23 +278,38 @@ private void initAndRecover() throws IOException {
.forEach(
consensusGroupId ->
resetPeerListWithoutThrow.accept(consensusGroupId, Collections.emptyList()));
final Exception resetFailure = peerListResetFailure.get();
if (resetFailure != null) {
throw new IOException(
ConsensusMessages.FAILED_TO_RESET_PEER_LIST_WHILE_START, resetFailure);
}
}
for (Map.Entry<ConsensusGroupId, IoTConsensusServerImpl> entry : stateMachineMap.entrySet()) {
final BiConsumer<ConsensusGroupId, IoTConsensusServerImpl> callback = onPeerRecovering;
if (callback != null) {
try {
callback.accept(entry.getKey(), entry.getValue());
} catch (RuntimeException e) {
throw new IOException(e);
}
}
}
if (correctPeerListBeforeStart != null) {
stateMachineMap.values().forEach(IoTConsensusServerImpl::markPeerConfigurationRecovered);
}
for (Map.Entry<ConsensusGroupId, IoTConsensusServerImpl> entry : stateMachineMap.entrySet()) {
try {
entry.getValue().start();
// Subscription metadata may have been loaded before any local consensus peers existed.
// Recovered peers must trigger the same binding as newly created peers, after startup has
// restored their indices and installed the pre-start WAL-retention guards.
notifyPeerCreated(entry.getKey(), entry.getValue());
} catch (RuntimeException e) {
throw new IOException(e);
} finally {
pendingPeerCreatedCallbacks.remove(entry.getKey());
}
}
stateMachineMap.forEach(
(consensusGroupId, consensus) -> {
try {
final BiConsumer<ConsensusGroupId, IoTConsensusServerImpl> callback = onPeerRecovering;
if (callback != null) {
callback.accept(consensusGroupId, consensus);
}
consensus.start();
// Subscription metadata may have been loaded before any local consensus peers existed.
// Recovered peers must trigger the same binding as newly created peers, after startup
// has restored their indices and installed the pre-start WAL-retention guards.
notifyPeerCreated(consensusGroupId, consensus);
} finally {
pendingPeerCreatedCallbacks.remove(consensusGroupId);
}
});
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,7 @@ public class IoTConsensusServerImpl {
private final Set<Peer> configuration = ConcurrentHashMap.newKeySet();
private final AtomicLong searchIndex;
private final Object retentionPublicationLock = new Object();
private boolean recoveringPeerConfiguration;
private final LogDispatcher logDispatcher;
private IoTConsensusConfig config;
private final ConsensusReqReader consensusReqReader;
Expand Down Expand Up @@ -249,6 +250,7 @@ public IoTConsensusServerImpl(
this.cacheQueueMap = new ConcurrentHashMap<>();
this.syncClientManager = syncClientManager;
this.configuration.addAll(configuration);
this.recoveringPeerConfiguration = configuration.isEmpty();
this.backgroundTaskService = backgroundTaskService;
this.config = config;
this.userDataTransferAuditHandler = userDataTransferAuditHandler;
Expand All @@ -271,12 +273,20 @@ public IStateMachine getStateMachine() {
}

public void start() {
checkAndUpdateIndex();
synchronized (retentionPublicationLock) {
checkAndUpdateIndex();
}
MetricService.getInstance().addMetricSet(this.ioTConsensusServerMetrics);
stateMachine.start();
logDispatcher.start();
}

void markPeerConfigurationRecovered() {
synchronized (retentionPublicationLock) {
recoveringPeerConfiguration = false;
}
}

public void stop() {
logDispatcher.stop();
persistLatestWriterMeta(true);
Expand Down Expand Up @@ -1486,6 +1496,9 @@ void checkAndUpdateIndex() {
*/
public void checkAndUpdateSafeDeletedSearchIndex() {
synchronized (retentionPublicationLock) {
if (recoveringPeerConfiguration) {
return;
}
if (configuration.isEmpty()) {
logger.error(IoTConsensusMessages.CONFIGURATION_EMPTY_UNEXPECTED);
return;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,11 @@ private void initLogSyncThreadPool() {

public synchronized void start() {
if (!threads.isEmpty()) {
threads.forEach(logDispatcherThread -> executorService.submit(logDispatcherThread));
threads.forEach(
logDispatcherThread -> {
logDispatcherThread.markSubmitted();
executorService.submit(logDispatcherThread);
});
}
}

Expand Down Expand Up @@ -140,6 +144,7 @@ public synchronized void addLogDispatcherThread(
initLogSyncThreadPool();
}
if (startNow) {
thread.markSubmitted();
executorService.submit(thread);
}
}
Expand Down Expand Up @@ -309,6 +314,7 @@ public class LogDispatcherThread implements Runnable {
private final LogDispatcherThreadMetrics logDispatcherThreadMetrics;

private final CountDownLatch runFinished = new CountDownLatch(1);
private volatile boolean submitted;
private volatile long lastIdleWriterSafeTimeBarrierSentTimeMs = 0L;

public LogDispatcherThread(Peer peer, IoTConsensusConfig config, long initialSyncIndex) {
Expand Down Expand Up @@ -385,8 +391,15 @@ public void stop() {
processStopped();
}

private void markSubmitted() {
submitted = true;
}

private void setStopped() {
stopped = true;
if (!submitted) {
runFinished.countDown();
}
}

private void processStopped() {
Expand Down
Loading
Loading