Repository: valkey-io/valkey
File: src/cluster_legacy.c (C)

Code before:

    /* Check if we reached the quorum. */
    if (server.cluster->failover_auth_count >= needed_quorum) {
        /* We have the quorum, we can finally failover the primary. */

        serverLog(LL_NOTICE, "Failover election won: I'm the new primary.");

        /* Update my configEpoch to the epoch of the election. */
        if (myself->configEpoch < server.cluster->failover_auth_epoch) {
            myself->configEpoch = server.cluster->failover_auth_epoch;
            serverLog(LL_NOTICE, "configEpoch set to %llu after successful failover",
                      (unsigned long long)myself->configEpoch);
        }

        /* Take responsibility for the cluster slots. */
        clusterFailoverReplaceYourPrimary();
    } else {
        clusterLogCantFailover(CLUSTER_CANT_FAILOVER_WAITING_VOTES);
    }
}

/* -----------------------------------------------------------------------------
 * CLUSTER replica migration
 *
 * Replica migration is the process that allows a replica of a primary that is
 * already covered by at least another replica, to "migrate" to a primary that
 * is orphaned, that is, left with no working replicas.
 * ------------------------------------------------------------------------- */

/* This function is responsible to decide if this replica should be migrated
 * to a different (orphaned) primary. It is called by the clusterCron() function
 * only if:
 *
 * 1) We are a replica node.
 * 2) It was detected that there is at least one orphaned primary in
 *    the cluster.
 * 3) We are a replica of one of the primaries with the greatest number of
 *    replicas.
 *
 * This checks are performed by the caller since it requires to iterate
 * the nodes anyway, so we spend time into clusterHandleReplicaMigration()
 * if definitely needed.
 *
 * The function is called with a pre-computed max_replicas, that is the max
 * number of working (not in FAIL state) replicas for a single primary.
 *
 * Additional conditions for migration are examined inside the function.
 */
void clusterHandleReplicaMigration(int max_replicas) {
    int j, ok_replicas = 0;
    clusterNode *my_primary = myself->replicaof, *target = NULL, *candidate = NULL;

Review comment (bug):
Curious - how did you arrive at 500?  Given that `CLUSTERMSG_TYPE_FAILOVER_AUTH_REQUEST` is broadcast and answered pretty much right away, unless the voter is busy, I would think the network round trip time between any two nodes should be significantly less than 50 ms for all deployments. I wonder if we could tighten it up a bit to like 250 or 200? 

Code after:
        data_age > (((mstime_t)server.repl_ping_replica_period * 1000) +
                    (server.cluster_node_timeout * server.cluster_replica_validity_factor))) {
        if (!manual_failover) {
            clusterLogCantFailover(CLUSTER_CANT_FAILOVER_DATA_AGE);
            return;
        }
    }

    /* If the previous failover attempt timeout and the retry time has
     * elapsed, we can setup a new one. */
    if (auth_age > auth_retry_time) {
        server.cluster->failover_auth_time = now +
                                             500 +           /* Fixed delay of 500 milliseconds, let FAIL msg propagate. */
                                             random() % 500; /* Random delay between 0 and 500 milliseconds. */
        server.cluster->failover_auth_count = 0;
        server.cluster->failover_auth_sent = 0;
        server.cluster->failover_auth_rank = clusterGetReplicaRank();
        /* We add another delay that is proportional to the replica rank.
         * Specifically 1 second * rank. This way replicas that have a probably
         * less updated replication offset, are penalized. */
        server.cluster->failover_auth_time += server.cluster->failover_auth_rank * 1000;
        /* We add another delay that is proportional to the failed primary rank.
         * Specifically 0.5 second * rank. This way those failed primaries will be
         * elected in rank to avoid the vote conflicts. */
        server.cluster->failover_failed_primary_rank = clusterGetFailedPrimaryRank();
        server.cluster->failover_auth_time += server.cluster->failover_failed_primary_rank * 500;
        /* However if this is a manual failover, no delay is needed. */
        if (server.cluster->mf_end) {
            server.cluster->failover_auth_time = now;
            server.cluster->failover_auth_rank = 0;
            /* Reset auth_age since it is outdated now and we can bypass the auth_timeout
             * check in the next state and start the election ASAP. */
            auth_age = 0;
        }
        serverLog(LL_NOTICE,
                  "Start of election delayed for %lld milliseconds "
                  "(rank #%d, primary rank #%d, offset %lld).",
                  server.cluster->failover_auth_time - now, server.cluster->failover_auth_rank,
                  server.cluster->failover_failed_primary_rank, replicationGetReplicaOffset());
        /* Now that we have a scheduled election, broadcast our offset
         * to all the other replicas so that they'll updated their offsets
         * if our offset is better. */
        clusterBroadcastPong(CLUSTER_BROADCAST_LOCAL_REPLICAS);

        /* Return ASAP if we can't start the election now. In a manual failover,
         * we can start the election immediately, so in this case we continue to
         * the next state without waiting for the next beforeSleep. */
        if (now < server.cluster->failover_auth_time) return;
    }

    /* It is possible that we received more updated offsets from other