Repository: danny-avila/LibreChat
File: packages/api/src/cluster/LeaderElection.ts (TypeScript)

Code before:
  static readonly LEADER_KEY = `${con.REDIS_KEY_PREFIX}${con.GLOBAL_PREFIX_SEPARATOR}LeadingServerUUID`;

  readonly UUID: string;
  private readonly LEADER_TTL: number; // seconds
  private readonly REFRESH_INTERVAL: number; // seconds
  private refreshTimer: NodeJS.Timeout | null = null;

  constructor(options?: { uuid?: string; leaderTTL?: number; refreshInterval?: number }) {
    this.UUID = options?.uuid || crypto.randomUUID();
    this.LEADER_TTL = options?.leaderTTL ?? 60;
    this.REFRESH_INTERVAL = options?.refreshInterval ?? 30;
    process.on('SIGTERM', () => this.resign());
    process.on('SIGINT', () => this.resign());
  }

  /**
   * Checks if this instance is the current leader.
   * If no leader exists, waits 1-3 seconds (randomized to avoid thundering herd) then attempts self-election.
   * Always returns true in non-Redis mode (single-instance deployment).
   */
  public async isLeader(): Promise<boolean> {
    if (!con.USE_REDIS || !keyvRedisClient) return true;

    try {
      const currentLeader = await keyvRedisClient.get(LeaderElection.LEADER_KEY);
      if (currentLeader === this.UUID) return true;
      if (currentLeader != null) return false;

      const delay = 1000 + Math.random() * 2000;
      await new Promise((resolve) => setTimeout(resolve, delay));
      return await this.electSelf();
    } catch (error) {
      logger.error('Failed to check leadership status:', error);
      return false;
    }
  }

  /**
   * Steps down from leadership by stopping the refresh timer and releasing the leader key.
   * Atomically deletes the leader key (only if we still own it) so another server can become leader immediately.
   */
  public async resign(): Promise<void> {
    if (this.refreshTimer) {
      clearInterval(this.refreshTimer);
      this.refreshTimer = null;
    }

    if (keyvRedisClient) {
      try {
        // Lua script for atomic check-and-delete (only delete if we still own it)
        const script = `

Review comment (bug):
There is a minor race condition here where the leadership expires after the check (let's say only 10ms are left on the TTL).

Give we refresh every 30s with a 60s TTL maybe we should check that there is at least 15s left or something like that

Code after:
  // We can't use Keyv namespace here because we need direct Redis access for atomic operations
  static readonly LEADER_KEY = `${cache.REDIS_KEY_PREFIX}${cache.GLOBAL_PREFIX_SEPARATOR}LeadingServerUUID`;
  private static _instance = new LeaderElection();

  readonly UUID: string = crypto.randomUUID();
  private refreshTimer: NodeJS.Timeout | undefined = undefined;

  // DO NOT create new instances of this class directly.
  // Use the exported isLeader() function which uses a singleton instance.
  constructor() {
    if (LeaderElection._instance) return LeaderElection._instance;

    process.on('SIGTERM', () => this.resign());
    process.on('SIGINT', () => this.resign());
    LeaderElection._instance = this;
  }

  /**
   * Checks if this instance is the current leader.
   * If no leader exists, waits upto 2 seconds (randomized to avoid thundering herd) then attempts self-election.
   * Always returns true in non-Redis mode (single-instance deployment).
   */
  public async isLeader(): Promise<boolean> {
    if (!cache.USE_REDIS) return true;

    try {
      const currentLeader = await LeaderElection.getLeaderUUID();
      // If we own the leadership lock, return true.
      // However, in case the leadership refresh retries have been exhausted, something has gone wrong.
      // This server is not considered the leader anymore, similar to a crash, to avoid split-brain scenario.
      if (currentLeader === this.UUID) return this.refreshTimer != null;
      if (currentLeader != null) return false; // someone holds leadership lock

      const delay = Math.random() * 2000;
      await new Promise((resolve) => setTimeout(resolve, delay));
      return await this.electSelf();
    } catch (error) {
      logger.error('Failed to check leadership status:', error);
      return false;
    }
  }

  /**
   * Steps down from leadership by stopping the refresh timer and releasing the leader key.
   * Atomically deletes the leader key (only if we still own it) so another server can become leader immediately.
   */
  public async resign(): Promise<void> {
    if (!cache.USE_REDIS) return;

    try {
      this.clearRefreshTimer();