Repository: valkey-io/valkey
File: src/server.c (C)

Code before:
                    serverCommunicateSystemd("STATUS=Error trying to save the DB, can't exit.\n");
                goto error;
            }
        }
    }

    /* Free the AOF manifest. */
    if (server.aof_manifest) aofManifestFree(server.aof_manifest);

    /* Fire the shutdown modules event. */
    moduleFireServerEvent(VALKEYMODULE_EVENT_SHUTDOWN, 0, NULL);

    /* Remove the pid file if possible and needed. */
    if (server.daemonize || server.pidfile) {
        serverLog(LL_NOTICE, "Removing the pid file.");
        unlink(server.pidfile);
    }

    /* Best effort flush of replica output buffers, so that we hopefully
     * send them pending writes. */
    flushReplicasOutputBuffers();

    if (server.auto_failover_on_shutdown && server.cluster_enabled && best_replica) {
        /* Sending a CLUSTER FAILOVER FORCE to the best replica. */
        const char *buf = "*3\r\n$7\r\nCLUSTER\r\n$8\r\nFAILOVER\r\n$5\r\nFORCE\r\n";
        if (connWrite(best_replica->conn, buf, strlen(buf)) == (int)strlen(buf)) {
            serverLog(LL_NOTICE, "Sending CLUSTER FAILOVER FORCE to replica %s succeeded.",
                      replicationGetReplicaName(best_replica));
        } else {
            serverLog(LL_WARNING, "Failed to send CLUSTER FAILOVER FORCE to replica: %s", strerror(errno));
        }
    }

    if (server.auto_failover_on_shutdown && server.cluster_enabled && !best_replica) {
        serverLog(LL_WARNING, "Unable to find a replica to perform an auto failover on shutdown.");
    }

    /* Close the listening sockets. Apparently this allows faster restarts. */
    closeListeningSockets(1);

    /* Handle cluster-related matters when shutdown. */
    if (server.cluster_enabled) clusterHandleServerShutdown();

    serverLog(LL_WARNING, "%s is now ready to exit, bye bye...", server.sentinel_mode ? "Sentinel" : "Valkey");
    return C_OK;

error:
    serverLog(LL_WARNING, "Errors trying to shut down the server. Check the logs for more information.");
    cancelShutdown();
    return C_ERR;
}

Review comment (bug):
This is in `finishShutdown()`, just before the primary does `exit()`.

Is there a risk that the written command has not yet been fully sent to the replica when the primary exits? If not, can we do it in an earlier shutdown stage and make sure the replica has received the command? The replica doesn't send any OK reply to the primary on the replication stream, but maybe the primary could wait for the failover auth request from the replica on the cluster bus before the primary shuts down?

Btw, maybe this code should be in `clusterHandleServerShutdown()` which is called a few lines below:

    /* Handle cluster-related matters when shutdown. */
    if (server.cluster_enabled) clusterHandleServerShutdown();

Code after:
 *
 * Flags:
 *
 * - SHUTDOWN_SAVE: Save a database dump even if the server is configured not to
 *   save any dump.
 *
 * - SHUTDOWN_NOSAVE: Don't save any database dump even if the server is
 *   configured to save one.
 *
 * - SHUTDOWN_NOW: Don't wait for replicas to catch up before shutting down.
 *
 * - SHUTDOWN_FORCE: Ignore errors writing AOF and RDB files on disk, which
 *   would normally prevent a shutdown.
 *
 * Unless SHUTDOWN_NOW is set and if any replicas are lagging behind, C_ERR is
 * returned and server.shutdown_mstime is set to a timestamp to allow a grace
 * period for the replicas to catch up. This is checked and handled by
 * serverCron() which completes the shutdown as soon as possible.
 *
 * If shutting down fails due to errors writing RDB or AOF files, C_ERR is
 * returned and an error is logged. If the flag SHUTDOWN_FORCE is set, these
 * errors are logged but ignored and C_OK is returned.
 *
 * On success, this function returns C_OK and then it's OK to call exit(0). */
int prepareForShutdown(client *c, int flags) {
    if (isShutdownInitiated()) return C_ERR;

    /* When SHUTDOWN is called while the server is loading a dataset in
     * memory we need to make sure no attempt is performed to save
     * the dataset on shutdown (otherwise it could overwrite the current DB
     * with half-read data).
     *
     * Also when in Sentinel mode clear the SAVE flag and force NOSAVE. */
    if (server.loading || server.sentinel_mode) flags = (flags & ~SHUTDOWN_SAVE) | SHUTDOWN_NOSAVE;

    server.shutdown_flags = flags;

    if (c != NULL) {
        sds client = catClientInfoShortString(sdsempty(), c, server.hide_user_data_from_log);
        serverLog(LL_NOTICE, "User requested shutdown... (user request from '%s')", client);
        sdsfree(client);
    } else {
        serverLog(LL_NOTICE, "User requested shutdown...");
    }
    if (server.supervised_mode == SUPERVISED_SYSTEMD) serverCommunicateSystemd("STOPPING=1\n");

    /* If we have any replicas, let them catch up the replication offset before
     * we shut down, to avoid data loss. */
    if (!(flags & SHUTDOWN_NOW) && server.shutdown_timeout != 0 && !isReadyToShutdown()) {
        server.shutdown_mstime = server.mstime + server.shutdown_timeout * 1000;
        if (!isPausedActions(PAUSE_ACTION_REPLICA)) sendGetackToReplicas();