Source:
<code>private void catchUp(JournalStateMachine stateMachine, LocalFirstRaftClient client) throws TimeoutException, InterruptedException { long startTime = System.currentTimeMillis(); CommonUtils.waitFor("snapshotting to finish", () -> !stateMachine.isSnapshotting(), WaitForOptions.defaults().setTimeoutMs(10 * Constants.MINUTE_MS)); while (true) { if (mPrimarySelector.getState() != PrimarySelector.State.PRIMARY) { return; } long lastAppliedSN = stateMachine.getLastAppliedSequenceNumber(); long gainPrimacySN = ThreadLocalRandom.current().nextLong(Long.MIN_VALUE, 0); LOG.info("Performing catchup. Last applied SN: {}. Catchup ID: {}", lastAppliedSN, gainPrimacySN); Exception ex; try { CompletableFuture<RaftClientReply> future = client.sendAsync( toRaftMessage(JournalEntry.newBuilder().setSequenceNumber(gainPrimacySN).build()), TimeDuration.valueOf(5, TimeUnit.SECONDS)); RaftClientReply reply = future.get(5, TimeUnit.SECONDS); ex = reply.getException(); } catch (TimeoutException | ExecutionException | IOException e) { ex = e; } if (ex != null) { LOG.info("Exception submitting term start entry: {}", ex.toString()); <START> Thread.sleep(100); <END> continue; } try { CommonUtils.waitFor("term start entry " + gainPrimacySN + " to be applied to state machine", () -> stateMachine.getLastPrimaryStartSequenceNumber() == gainPrimacySN, WaitForOptions.defaults() .setInterval(Constants.SECOND_MS) .setTimeoutMs(5 * Constants.SECOND_MS)); } catch (TimeoutException e) { LOG.info(e.toString()); continue; } CommonUtils.sleepMs(2 * mConf.getElectionTimeoutMs()); if (stateMachine.getLastAppliedSequenceNumber() != lastAppliedSN || stateMachine.getLastPrimaryStartSequenceNumber() != gainPrimacySN) { continue; } LOG.info("Caught up in {}ms. Last sequence number from previous term: {}.", System.currentTimeMillis() - startTime, stateMachine.getLastAppliedSequenceNumber()); return; } }</code><technical_language>this configurable? I if 100ms is short long. This means retry 10x forever? frequent</technical_language>

Target:
private void catchUp(JournalStateMachine stateMachine, LocalFirstRaftClient client) throws TimeoutException, InterruptedException { long startTime = System.currentTimeMillis(); long waitBeforeRetry = ServerConfiguration.global() .getMs(PropertyKey.MASTER_EMBEDDED_JOURNAL_CATCHUP_RETRY_WAIT); CommonUtils.waitFor("snapshotting to finish", () -> !stateMachine.isSnapshotting(), WaitForOptions.defaults().setTimeoutMs(10 * Constants.MINUTE_MS)); while (true) { if (mPrimarySelector.getState() != PrimarySelector.State.PRIMARY) { return; } long lastAppliedSN = stateMachine.getLastAppliedSequenceNumber(); long gainPrimacySN = ThreadLocalRandom.current().nextLong(Long.MIN_VALUE, 0); LOG.info("Performing catchup. Last applied SN: {}. Catchup ID: {}", lastAppliedSN, gainPrimacySN); Exception ex; try { CompletableFuture<RaftClientReply> future = client.sendAsync( toRaftMessage(JournalEntry.newBuilder().setSequenceNumber(gainPrimacySN).build()), TimeDuration.valueOf(5, TimeUnit.SECONDS)); RaftClientReply reply = future.get(5, TimeUnit.SECONDS); ex = reply.getException(); } catch (TimeoutException | ExecutionException | IOException e) { ex = e; } if (ex != null) { LOG.info("Exception submitting term start entry: {}", ex.toString()); Thread.sleep(waitBeforeRetry); continue; } try { CommonUtils.waitFor("term start entry " + gainPrimacySN + " to be applied to state machine", () -> stateMachine.getLastPrimaryStartSequenceNumber() == gainPrimacySN, WaitForOptions.defaults() .setInterval(Constants.SECOND_MS) .setTimeoutMs(5 * Constants.SECOND_MS)); } catch (TimeoutException e) { LOG.info(e.toString()); continue; } CommonUtils.sleepMs(2 * mConf.getElectionTimeoutMs()); if (stateMachine.getLastAppliedSequenceNumber() != lastAppliedSN || stateMachine.getLastPrimaryStartSequenceNumber() != gainPrimacySN) { continue; } LOG.info("Caught up in {}ms. Last sequence number from previous term: {}.", System.currentTimeMillis() - startTime, stateMachine.getLastAppliedSequenceNumber()); return; } }