Skill: adriel1006/discord-voice-deepgram (version 1.0.0)

--- name: deepgram-discord-voice description: Voice-channel conversations in Discord using Deepgram streaming STT + low-latency TTS metadata: clawdbot: config: requiredEnv: - DISCORD_TOKEN - DEEPGRAM_API_KEY optionalEnv: [] example: | { "plugins": { "entries": { "deepgram-discord-voice": { "enabled": true, "config": { "streamingSTT": true, "streamingTTS": true, "ttsVoice": "aura-2-thalia-en", "vadSensitivity": "medium", "bargeIn": true, "primaryUser": "atechy", "allowVoiceSwitch": true, "wakeWord": "openclaw", "deepgram": { "sttModel": "nova-2", "language": "en-US" } } } } } } --- # Deepgram Discord Voice (Clawdbot/OpenClaw Plugin) This plugin lets you talk to your agent **only from a Discord voice channel**. Pipeline (low latency): - Discord voice audio → **Deepgram streaming STT** (WebSocket) - Transcript → your agent - Agent reply → **Deepgram TTS** (`/v1/speak` streamed HTTP Ogg/Opus) - Audio played back into the voice channel ## Requirements - A Discord bot token (`DISCORD_TOKEN`) - A Deepgram API key (`DEEPGRAM_API_KEY`) - Discord bot permissions in your server: - **Connect** - **Speak** - **Use Voice Activity** ## Install ### Option A: Install from ClawHub 1. In your OpenClaw/Clawdbot dashboard, open **Skills/Plugins**. 2. Add/install **deepgram-discord-voice**. 3. Set the required environment variables. ### Option B: Manual install 1. Copy this folder into your extensions/plugins directory. 2. Run: ```bash npm install ``` 3. Restart OpenClaw/Clawdbot. ## Configuration ### Key settings - `primaryUser` (recommended): Who the bot listens to by default. - Best: your **Discord user ID** (numeric) - Also supported: username/display name (e.g., `atechy`) if unique in-channel - `allowVoiceSwitch`: If `true`, the primary user can switch who is allowed by voice. - `wakeWord`: Prefix for voice control commands. Default: `openclaw`. - `deepgram.sttModel`: Default `nova-2`. - `deepgram.language`: Optional BCP‑47 language tag (e.g., `en-US`, `es`, `es-EC`). - `ttsVoice`: Deepgram Aura voice model (e.g., `aura-2-thalia-en`). ### Example config ```json5 { "plugins": { "entries": { "deepgram-discord-voice": { "enabled": true, "config": { "streamingSTT": true, "streamingTTS": true, "primaryUser": "atechy", "allowVoiceSwitch": true, "wakeWord": "openclaw", "ttsVoice": "aura-2-thalia-en", "vadSensitivity": "medium", "bargeIn": true, "deepgram": { "sttModel": "nova-2", "language": "en-US" } } } } } } ``` ## Usage ### Join a voice channel Use the plugin tool or slash command (depends on your OpenClaw setup): - Join: `action=join` with the `channelId` - Leave: `action=leave` ### Talk (voice channel) Once the bot is connected, just speak. ### Safeguard: only listen to you (default) When `primaryUser` is set, the plugin will only listen to that user unless you allow someone else. ### Let someone else talk (voice commands) As the primary user, say: - `openclaw allow <name>` - `openclaw listen to <name>` To lock it back: - `openclaw only me` - `openclaw reset` ### Switch via tool actions (optional) - `allow_speaker` with `user` (id / @mention / name) - `only_me` - `status` ## Notes - Lowest latency comes from `streamingSTT=true` and `streamingTTS=true`. - Deepgram TTS is streamed over HTTP in **Ogg/Opus** so Discord can play it immediately.

path: clawdbot.plugin.json
content: {
  "id": "deepgram-discord-voice",
  "name": "Deepgram Discord Voice",
  "description": "Low-latency Discord voice conversations using Deepgram streaming STT + TTS",
  "configSchema": {
    "type": "object",
    "additionalProperties": false,
    "properties": {
      "enabled": {
        "type": "boolean",
        "default": true
      },
      "streamingSTT": {
        "type": "boolean",
        "default": true,
        "description": "Use streaming STT over WebSocket (recommended for lowest latency)"
      },
      "streamingTTS": {
        "type": "boolean",
        "default": true,
        "description": "Use low-latency TTS (HTTP streamed Ogg/Opus output)"
      },
      "ttsVoice": {
        "type": "string",
        "default": "aura-2-thalia-en",
        "description": "Deepgram Aura voice model (e.g., aura-2-thalia-en)"
      },
      "vadSensitivity": {
        "type": "string",
        "enum": [
          "low",
          "medium",
          "high"
        ],
        "default": "medium"
      },
      "bargeIn": {
        "type": "boolean",
        "default": true,
        "description": "Stop speaking when user starts talking"
      },
      "model": {
        "type": "string",
        "description": "LLM model for voice responses (e.g., anthropic/claude-3-5-haiku-latest)"
      },
      "thinkLevel": {
        "type": "string",
        "enum": [
          "off",
          "low",
          "medium",
          "high"
        ],
        "default": "off",
        "description": "Thinking level for voice responses (lower = faster)"
      },
      "allowedUsers": {
        "type": "array",
        "items": {
          "type": "string"
        },
        "default": []
      },
      "silenceThresholdMs": {
        "type": "number",
        "default": 1000
      },
      "minAudioMs": {
        "type": "number",
        "default": 300
      },
      "maxRecordingMs": {
        "type": "number",
        "default": 30000
      },
      "heartbeatIntervalMs": {
        "type": "number",
        "default": 30000,
        "description": "Connection health check interval in ms"
      },
      "autoJoinChannel": {
        "type": "string",
        "description": "Voice channel ID to auto-join on startup"
      },
      "deepgram": {
        "type": "object",
        "additionalProperties": false,
        "properties": {
          "apiKey": {
            "type": "string"
          },
          "sttModel": {
            "type": "string",
            "default": "nova-2"
          },
          "language": {
            "type": "string",
            "description": "BCP-47 language tag (optional). Example: en-US, es"
          }
        }
      },
      "primaryUser": {
        "type": "string",
        "description": "Discord user ID or name allowed by default (best: user ID). If set, only this user can talk unless switched."
      },
      "allowVoiceSwitch": {
        "type": "boolean",
        "default": true,
        "description": "Allow voice commands (from primary user) to switch who the bot listens to"
      },
      "wakeWord": {
        "type": "string",
        "default": "openclaw",
        "description": "Wake word for voice commands like 'openclaw allow John'"
      }
    }
  },
  "uiHints": {
    "enabled": {
      "label": "Enable Discord Voice",
      "help": "Enable real-time voice conversations in Discord voice channels"
    },
    "streamingSTT": {
      "label": "Streaming STT",
      "help": "Use Deepgram WebSocket STT for lowest latency"
    },
    "streamingTTS": {
      "label": "Low-latency TTS",
      "help": "Stream Deepgram TTS audio as it is generated"
    },
    "ttsVoice": {
      "label": "Deepgram TTS Voice Model",
      "help": "Example: aura-2-thalia-en (see Deepgram Voices and Languages)"
    },
    "vadSensitivity": {
      "label": "VAD Sensitivity",
      "help": "Voice activity detection sensitivity (low/medium/high)"
    },
    "bargeIn": {},
    "allowedUsers": {
      "label": "Allowed Users",
      "help": "Discord user IDs allowed to use voice (empty = all allowed)"
    },
    "deepgram.apiKey": {
      "label": "Deepgram API Key",
      "sensitive": true
    },
    "deepgram.sttModel": {
      "label": "Deepgram STT Model",
      "help": "Example: nova-2, nova-3, etc."
    },
    "deepgram.language": {
      "label": "Language (optional)",
      "help": "BCP-47 tag like en-US, es, es-EC"
    },
    "primaryUser": {
      "label": "Primary Speaker",
      "help": "Discord user ID (recommended) or name. Default speaker the bot listens to."
    },
    "allowVoiceSwitch": {
      "label": "Allow Voice Speaker Switching",
      "help": "Primary speaker can say voice commands to let someone else speak"
    },
    "wakeWord": {
      "label": "Wake Word",
      "help": "Example: openclaw. Use: 'openclaw allow <name>' or 'openclaw only me'."
    }
  }
}

path: index.ts
content: /**
 * Deepgram Discord Voice Plugin for Clawdbot
 * 
 * Enables real-time voice conversations in Discord voice channels.
 * 
 * Features:
 * - Join/leave voice channels via slash commands (/voice join, /voice leave)
 * - Listen to user speech with VAD (Voice Activity Detection)
 * - Speech-to-text via Whisper API or Deepgram
 * - Routes transcribed text through Clawdbot agent
 * - Text-to-speech via OpenAI or ElevenLabs
 * - Plays audio responses back to the voice channel
 */

import crypto from "node:crypto";
import { Type } from "@sinclair/typebox";
import { Client, GatewayIntentBits, type VoiceBasedChannel, type GuildMember } from "discord.js";

import { parseConfig, type DiscordVoiceConfig } from "./src/config.js";
import { VoiceConnectionManager } from "./src/voice-connection.js";
import { loadCoreAgentDeps, type CoreConfig } from "./src/core-bridge.js";

interface PluginApi {
  pluginConfig: unknown;
  config: unknown;
  logger: {
    info(msg: string): void;
    warn(msg: string): void;
    error(msg: string): void;
    debug?(msg: string): void;
  };
  runtime: {
    discord?: {
      getClient(accountId?: string): Client | null;
    };
    agent?: {
      chat(params: {
        sessionKey: string;
        message: string;
        channel?: string;
        senderId?: string;
      }): Promise<{ text: string }>;
    };
  };
  registerGatewayMethod(
    name: string,
    handler: (ctx: { params: unknown; respond: (ok: boolean, payload?: unknown) => void }) => void | Promise<void>
  ): void;
  registerTool(tool: {
    name: string;
    label: string;
    description: string;
    parameters: unknown;
    execute: (toolCallId: string, params: unknown) => Promise<{
      content: Array<{ type: string; text: string }>;
      details?: unknown;
    }>;
  }): void;
  registerService(service: {
    id: string;
    start: () => Promise<void> | void;
    stop: () => Promise<void> | void;
  }): void;
  registerCli(
    register: (ctx: { program: unknown }) => void,
    opts?: { commands: string[] }
  ): void;
}

const VoiceToolSchema = Type.Union([
  Type.Object({
    action: Type.Literal("join"),
    channelId: Type.String({ description: "Voice channel ID to join" }),
    guildId: Type.Optional(Type.String({ description: "Guild ID (optional if in guild context)" })),
  }),
  Type.Object({
    action: Type.Literal("leave"),
    guildId: Type.Optional(Type.String({ description: "Guild ID to leave voice in (optional)" })),
  }),
  Type.Object({
    action: Type.Literal("speak"),
    text: Type.String({ description: "Text to speak in the voice channel" }),
    guildId: Type.Optional(Type.String({ description: "Guild ID (optional)" })),
  }),
  Type.Object({
    action: Type.Literal("status"),
  }),
  Type.Object({
    action: Type.Literal("allow_speaker"),
    user: Type.String({ description: "User ID, mention, or name to allow in the current voice channel" }),
    guildId: Type.Optional(Type.String({ description: "Guild ID (optional)" })),
  }),
  Type.Object({
    action: Type.Literal("only_me"),
    guildId: Type.Optional(Type.String({ description: "Guild ID (optional)" })),
  }),
]);

const discordVoicePlugin = {
  id: "deepgram-discord-voice",
  name: "Deepgram Discord Voice",
  description: "Real-time voice conversations in Discord voice channels",

  configSchema: {
    parse(value: unknown): DiscordVoiceConfig {
      return parseConfig(value);
    },
  },

  register(api: PluginApi) {
    const cfg = parseConfig(api.pluginConfig);
    let voiceManager: VoiceConnectionManager | null = null;
    let discordClient: Client | null = null;
    let clientReady = false;

    if (!cfg.enabled) {
      api.logger.info("[deepgram-discord-voice] Plugin disabled");
      return;
    }

    // Get Discord token from main config
    const mainConfig = api.config as { discord?: { token?: string }, channels?: { discord?: { token?: string } } };
    const discordToken = mainConfig?.channels?.discord?.token || mainConfig?.discord?.token;
    
    if (!discordToken) {
      api.logger.error("[deepgram-discord-voice] No Discord token found in config. Plugin requires discord.token to be configured.");
      return;
    }

    // Create our own Discord client with voice intents
    discordClient = new Client({
      intents: [
        GatewayIntentBits.Guilds,
        GatewayIntentBits.GuildVoiceStates,
        GatewayIntentBits.GuildMessages,
      ],
    });

    discordClient.once("ready", async () => {
      clientReady = true;
      api.logger.info(`[deepgram-discord-voice] Discord client ready as ${discordClient?.user?.tag}`);
      
      // Set bot user ID on voice manager for echo filtering
      if (discordClient?.user?.id && voiceManager) {
        voiceManager.setBotUserId(discordClient.user.id);
      }
      
      // Auto-join channel if configured
      if (cfg.autoJoinChannel) {
        try {
          api.logger.info(`[deepgram-discord-voice] Auto-joining channel ${cfg.autoJoinChannel}`);
          // Wait a moment for everything to initialize
          await new Promise(resolve => setTimeout(resolve, 2000));
          
          const channel = await discordClient!.channels.fetch(cfg.autoJoinChannel);
          if (channel && channel.isVoiceBased()) {
            const vm = ensureVoiceManager();
            await vm.join(channel as VoiceBasedChannel);
            api.logger.info(`[deepgram-discord-voice] Auto-joined voice channel: ${channel.name}`);
          } else {
            api.logger.warn(`[deepgram-discord-voice] Auto-join channel ${cfg.autoJoinChannel} is not a voice channel`);
          }
        } catch (error) {
          api.logger.error(`[deepgram-discord-voice] Failed to auto-join: ${error instanceof Error ? error.message : String(error)}`);
        }
      }
    });

    discordClient.login(discordToken).catch((err) => {
      api.logger.error(`[deepgram-discord-voice] Failed to login: ${err instanceof Error ? err.message : String(err)}`);
    });

    /**
     * Handle transcribed speech - route to agent and get response
     */
    async function handleTranscript(
      userId: string,
      guildId: string,
      channelId: string,
      text: string
    ): Promise<string> {
      api.logger.info(`[deepgram-discord-voice] Processing transcript from ${userId}: "${text}"`);

      try {
        const deps = await loadCoreAgentDeps();
        if (!deps) {
          api.logger.error("[deepgram-discord-voice] Could not load core dependencies");
          return "I'm having trouble connecting to my brain right now.";
        }

        const coreConfig = api.config as CoreConfig;
        const agentId = "main";
        
        // Build session key based on guild
        const sessionKey = `discord:voice:${guildId}`;
        
        // Resolve paths
        const storePath = deps.resolveStorePath(coreConfig.session?.store, { agentId });
        const agentDir = deps.resolveAgentDir(coreConfig, agentId);
        const workspaceDir = deps.resolveAgentWorkspaceDir(coreConfig, agentId);

        // Ensure workspace exists
        await deps.ensureAgentWorkspace({ dir: workspaceDir });

        // Load or create session entry
        const sessionStore = deps.loadSessionStore(storePath);
        const now = Date.now();
        type SessionEntry = { sessionId: string; updatedAt: number };
        let sessionEntry = sessionStore[sessionKey] as SessionEntry | undefined;

        if (!sessionEntry) {
          sessionEntry = {
            sessionId: crypto.randomUUID(),
            updatedAt: now,
          };
          sessionStore[sessionKey] = sessionEntry;
          await deps.saveSessionStore(storePath, sessionStore);
        }

        const sessionId = sessionEntry.sessionId;
        const sessionFile = deps.resolveSessionFilePath(sessionId, sessionEntry, { agentId });

        // Resolve model - use voice-specific model if configured, otherwise default
        const modelRef = cfg.model || `${deps.DEFAULT_PROVIDER}/${deps.DEFAULT_MODEL}`;
        const slashIndex = modelRef.indexOf("/");
        const provider = slashIndex === -1 ? deps.DEFAULT_PROVIDER : modelRef.slice(0, slashIndex);
        const model = slashIndex === -1 ? modelRef : modelRef.slice(slashIndex + 1);

        // Resolve thinking level - use voice-specific level if configured (default to "off" for speed)
        const thinkLevel = cfg.thinkLevel || "off";

        // Resolve agent identity
        const identity = deps.resolveAgentIdentity(coreConfig, agentId);
        const agentName = identity?.name?.trim() || "assistant";

        const extraSystemPrompt = `You are ${agentName}, speaking in a Discord voice channel. Keep responses brief and conversational (1-2 sentences max). Be natural and friendly. You have access to all your normal tools and skills. The user's Discord ID is ${userId}.`;

        const timeoutMs = deps.resolveAgentTimeoutMs({ cfg: coreConfig });
        const runId = `discord-voice:${guildId}:${Date.now()}`;

        const result = await deps.runEmbeddedPiAgent({
          sessionId,
          sessionKey,
          messageProvider: "discord",
          sessionFile,
          workspaceDir,
          config: coreConfig,
          prompt: text,
          provider,
          model,
          thinkLevel,
          verboseLevel: "off",
          timeoutMs,
          runId,
          // lane: "discord-voice",  // Removed - was possibly restricting tool access
          extraSystemPrompt,
          agentDir,
        });

        // Extract text from payloads
        const texts = (result.payloads ?? [])
          .filter((p) => p.text && !p.isError)
          .map((p) => p.text?.trim())
          .filter(Boolean);

        return texts.join(" ") || "";
      } catch (error) {
        api.logger.error(`[deepgram-discord-voice] Agent chat error: ${error instanceof Error ? error.message : String(error)}`);
        return "I'm sorry, I encountered an error processing your request.";
      }
    }

    /**
     * Ensure voice manager is initialized
     */
    function ensureVoiceManager(): VoiceConnectionManager {
      if (!voiceManager) {
        const botUserId = discordClient?.user?.id;
        voiceManager = new VoiceConnectionManager(cfg, api.logger, handleTranscript, botUserId);
      }
      return voiceManager;
    }

    /**
     * Get Discord client
     */
    function getDiscordClient(): Client | null {
      if (!clientReady) {
        api.logger.warn("[deepgram-discord-voice] Discord client not ready yet");
        return null;
      }
      return discordClient;
    }

    // Register Gateway RPC methods
    api.registerGatewayMethod("discord-voice.join", async ({ params, respond }) => {
      try {
        const p = params as { channelId?: string; guildId?: string } | null;
        const channelId = p?.channelId;
        const guildId = p?.guildId;

        if (!channelId) {
          respond(false, { error: "channelId required" });
          return;
        }

        const client = getDiscordClient();
        if (!client) {
          respond(false, { error: "Discord client not available" });
          return;
        }

        const channel = await client.channels.fetch(channelId);
        if (!channel || !("guild" in channel) || !channel.isVoiceBased()) {
          respond(false, { error: "Invalid voice channel" });
          return;
        }

        const vm = ensureVoiceManager();
        const session = await vm.join(channel as VoiceBasedChannel);
        
        respond(true, {
          joined: true,
          guildId: session.guildId,
          channelId: session.channelId,
        });
      } catch (error) {
        respond(false, { error: error instanceof Error ? error.message : String(error) });
      }
    });

    api.registerGatewayMethod("discord-voice.leave", async ({ params, respond }) => {
      try {
        const p = params as { guildId?: string } | null;
        let guildId = p?.guildId;

        const vm = ensureVoiceManager();
        
        // If no guildId provided, leave all
        if (!guildId) {
          const sessions = vm.getAllSessions();
          if (sessions.length === 0) {
            respond(true, { left: false, reason: "Not in any voice channel" });
            return;
          }
          guildId = sessions[0].guildId;
        }

        const left = await vm.leave(guildId);
        respond(true, { left, guildId });
      } catch (error) {
        respond(false, { error: error instanceof Error ? error.message : String(error) });
      }
    });

    api.registerGatewayMethod("discord-voice.speak", async ({ params, respond }) => {
      try {
        const p = params as { text?: string; guildId?: string } | null;
        const text = p?.text;
        let guildId = p?.guildId;

        if (!text) {
          respond(false, { error: "text required" });
          return;
        }

        const vm = ensureVoiceManager();
        
        if (!guildId) {
          const sessions = vm.getAllSessions();
          if (sessions.length === 0) {
            respond(false, { error: "Not in any voice channel" });
            return;
          }
          guildId = sessions[0].guildId;
        }

        await vm.speak(guildId, text);
        respond(true, { spoken: true });
      } catch (error) {
        respond(false, { error: error instanceof Error ? error.message : String(error) });
      }
    });

    api.registerGatewayMethod("discord-voice.status", async ({ respond }) => {
      try {
        const vm = ensureVoiceManager();
        const sessions = vm.getAllSessions().map((s) => ({
          guildId: s.guildId,
          channelId: s.channelId,
          speaking: s.speaking,
          usersListening: s.userAudioStates.size,
        }));
        respond(true, { sessions });
      } catch (error) {
        respond(false, { error: error instanceof Error ? error.message : String(error) });
      }
    });

    // Register agent tool
    api.registerTool({
      name: "discord_voice",
      label: "Discord Voice",
      description: "Control Discord voice channel - join, leave, speak, status, and speaker permissions",
      parameters: VoiceToolSchema,
      async execute(_toolCallId, params) {
        const json = (payload: unknown) => ({
          content: [{ type: "text", text: JSON.stringify(payload, null, 2) }],
          details: payload,
        });

        try {
          const p = params as { action: string; channelId?: string; guildId?: string; text?: string; user?: string };
          const vm = ensureVoiceManager();
          const client = getDiscordClient();

          switch (p.action) {
            case "join": {
              if (!p.channelId) throw new Error("channelId required");
              if (!client) throw new Error("Discord client not available");
              
              const channel = await client.channels.fetch(p.channelId);
              if (!channel || !("guild" in channel) || !channel.isVoiceBased()) {
                throw new Error("Invalid voice channel");
              }
              
              const session = await vm.join(channel as VoiceBasedChannel);
              return json({ joined: true, guildId: session.guildId, channelId: session.channelId });
            }

            case "leave": {
              let guildId = p.guildId;
              if (!guildId) {
                const sessions = vm.getAllSessions();
                if (sessions.length === 0) {
                  return json({ left: false, reason: "Not in any voice channel" });
                }
                guildId = sessions[0].guildId;
              }
              const left = await vm.leave(guildId);
              return json({ left, guildId });
            }

            case "speak": {
              if (!p.text) throw new Error("text required");
              let guildId = p.guildId;
              if (!guildId) {
                const sessions = vm.getAllSessions();
                if (sessions.length === 0) {
                  throw new Error("Not in any voice channel");
                }
                guildId = sessions[0].guildId;
              }
              await vm.speak(guildId, p.text);
              return json({ spoken: true });
            }

            case "status": {
              const sessions = vm.getAllSessions().map((s) => {
                let speaker: any = {};
                try { speaker = vm.getSpeakerStatus(s.guildId); } catch {}
                return {
                  guildId: s.guildId,
                  channelId: s.channelId,
                  speaking: s.speaking,
                  usersListening: s.userAudioStates.size,
                  primaryUserId: speaker.primaryUserId,
                  activeSpeakerId: speaker.activeSpeakerId,
                };
              });
              return json({ sessions });
            }


            case "allow_speaker": {
              const guildId = p.guildId || vm.getAllSessions()[0]?.guildId;
              if (!guildId) throw new Error("No active voice session (join first) or provide guildId");
              if (!p.user) throw new Error("user required");
              const speaker = vm.setActiveSpeaker(guildId, p.user);
              return json({ ok: true, ...speaker });
            }

            case "only_me": {
              const guildId = p.guildId || vm.getAllSessions()[0]?.guildId;
              if (!guildId) throw new Error("No active voice session (join first) or provide guildId");
              const speaker = vm.clearActiveSpeaker(guildId);
              return json({ ok: true, ...speaker });
            }

            default:
              throw new Error(`Unknown action: ${p.action}`);
          }
        } catch (error) {
          return json({ error: error instanceof Error ? error.message : String(error) });
        }
      },
    });

    // Register CLI commands
    api.registerCli(
      ({ program }) => {
        // eslint-disable-next-line @typescript-eslint/no-explicit-any
        const prog = program as any;

        const voiceCmd = prog.command("voice").description("Discord voice channel commands");

        voiceCmd
          .command("join")
          .description("Join a Discord voice channel")
          .argument("<channelId>", "Voice channel ID")
          .action(async (channelId: string) => {
            const client = getDiscordClient();
            if (!client) {
              console.error("Discord client not available");
              return;
            }

            try {
              const channel = await client.channels.fetch(channelId);
              if (!channel || !("guild" in channel) || !channel.isVoiceBased()) {
                console.error("Invalid voice channel");
                return;
              }

              const vm = ensureVoiceManager();
              const session = await vm.join(channel as VoiceBasedChannel);
              console.log(`Joined voice channel in guild ${session.guildId}`);
            } catch (error) {
              console.error(`Failed to join: ${error instanceof Error ? error.message : String(error)}`);
            }
          });

        voiceCmd
          .command("leave")
          .description("Leave the current voice channel")
          .option("-g, --guild <guildId>", "Guild ID")
          .action(async (opts: { guild?: string }) => {
            const vm = ensureVoiceManager();
            const guildId = opts.guild || vm.getAllSessions()[0]?.guildId;
            
            if (!guildId) {
              console.log("Not in any voice channel");
              return;
            }

            const left = await vm.leave(guildId);
            console.log(left ? `Left voice channel in guild ${guildId}` : "Failed to leave");
          });

        voiceCmd
          .command("status")
          .description("Show voice connection status")
          .action(() => {
            const vm = ensureVoiceManager();
            const sessions = vm.getAllSessions();

            if (sessions.length === 0) {
              console.log("Not connected to any voice channels");
              return;
            }

            for (const s of sessions) {
              console.log(`Guild: ${s.guildId}`);
              console.log(`  Channel: ${s.channelId}`);
              console.log(`  Speaking: ${s.speaking}`);
              console.log(`  Users listening: ${s.userAudioStates.size}`);
            }
          });
      },
      { commands: ["voice"] }
    );

    // Register background service
    api.registerService({
      id: "deepgram-discord-voice",
      start: async () => {
        api.logger.info("[deepgram-discord-voice] Service started");
      },
      stop: async () => {
        if (voiceManager) {
          await voiceManager.destroy();
          voiceManager = null;
        }
        api.logger.info("[deepgram-discord-voice] Service stopped");
      },
    });
  },
};

export default discordVoicePlugin;


path: package.json
content: {
  "name": "@clawdbot/deepgram-discord-voice",
  "version": "0.2.0",
  "type": "module",
  "description": "Discord voice channel plugin for Clawdbot/OpenClaw using Deepgram streaming STT + low-latency TTS",
  "main": "index.ts",
  "author": "Discord Voice Plugin Contributors",
  "license": "MIT",
  "repository": {
    "type": "git",
    "url": "https://github.com/openclaw/clawhub"
  },
  "keywords": [
    "clawdbot",
    "discord",
    "voice",
    "speech-to-text",
    "text-to-speech",
    "ai",
    "deepgram"
  ],
  "dependencies": {
    "@discordjs/voice": "^0.18.0",
    "discord.js": "^14.16.3",
    "libsodium-wrappers": "^0.7.15",
    "opusscript": "^0.0.8",
    "prism-media": "^1.3.5",
    "ws": "^8.18.0"
  },
  "clawdbot": {
    "extensions": [
      "./index.ts"
    ]
  },
  "devDependencies": {
    "@sinclair/typebox": "^0.34.48",
    "@types/node": "^25.0.10",
    "@types/ws": "^8.18.0",
    "typescript": "^5.9.3"
  }
}

path: tsconfig.json
content: {
  "compilerOptions": {
    "target": "ES2022",
    "module": "NodeNext",
    "moduleResolution": "NodeNext",
    "esModuleInterop": true,
    "strict": true,
    "skipLibCheck": true,
    "noEmit": true,
    "declaration": false,
    "resolveJsonModule": true,
    "lib": ["ES2022"]
  },
  "include": ["*.ts", "src/**/*.ts"],
  "exclude": ["node_modules"]
}


path: _meta.json
content: ownerId: kn738y5j6ep0rq1e9efavpzkth7zyh41
slug: deepgram-discord-voice
version: 0.2.0
publishedAt: 1769456717738

path: src/config.ts
content: /**
 * Deepgram Discord Voice - Plugin Configuration
 */

export interface DeepgramConfig {
  apiKey?: string;
  sttModel?: string;
  language?: string; // BCP-47 tag, optional
}

export interface DiscordVoiceConfig {
  enabled: boolean;

  // Low-latency paths
  streamingSTT: boolean;
  streamingTTS: boolean;

  // Deepgram Aura TTS voice model name (example: "aura-2-thalia-en")
  ttsVoice: string;

  // Voice UX
  vadSensitivity: "low" | "medium" | "high";
  bargeIn: boolean;
  allowedUsers: string[];

  // Speaker safelist / switching
  primaryUser?: string; // Discord user ID or name (best: user ID)
  allowVoiceSwitch: boolean; // allow voice commands to switch active speaker
  wakeWord: string; // word to start commands, e.g. "openclaw"

  // Recording / endpointing
  silenceThresholdMs: number;
  minAudioMs: number;
  maxRecordingMs: number;

  // Optional features
  autoJoinChannel?: string;
  heartbeatIntervalMs?: number;

  // LLM response tuning (fast models recommended)
  model?: string;
  thinkLevel?: string;

  deepgram?: DeepgramConfig;
}

export const DEFAULT_CONFIG: DiscordVoiceConfig = {
  enabled: true,
  streamingSTT: true,
  streamingTTS: true,
  ttsVoice: "aura-2-thalia-en",
  vadSensitivity: "medium",
  bargeIn: true,
  allowedUsers: [],
  primaryUser: undefined,
  allowVoiceSwitch: true,
  wakeWord: "openclaw",
  silenceThresholdMs: 1000,
  minAudioMs: 300,
  maxRecordingMs: 30_000,
  heartbeatIntervalMs: 30_000,
};

export function parseConfig(raw: unknown): DiscordVoiceConfig {
  if (!raw || typeof raw !== "object") return { ...DEFAULT_CONFIG };
  const obj = raw as Record<string, unknown>;

  const deepgramRaw = obj.deepgram && typeof obj.deepgram === "object" ? (obj.deepgram as Record<string, unknown>) : undefined;

  return {
    enabled: typeof obj.enabled === "boolean" ? obj.enabled : DEFAULT_CONFIG.enabled,
    streamingSTT: typeof obj.streamingSTT === "boolean" ? obj.streamingSTT : DEFAULT_CONFIG.streamingSTT,
    streamingTTS: typeof obj.streamingTTS === "boolean" ? obj.streamingTTS : DEFAULT_CONFIG.streamingTTS,
    ttsVoice: typeof obj.ttsVoice === "string" && obj.ttsVoice.trim() ? obj.ttsVoice.trim() : DEFAULT_CONFIG.ttsVoice,
    vadSensitivity: (obj.vadSensitivity === "low" || obj.vadSensitivity === "medium" || obj.vadSensitivity === "high")
      ? (obj.vadSensitivity as "low" | "medium" | "high")
      : DEFAULT_CONFIG.vadSensitivity,
    bargeIn: typeof obj.bargeIn === "boolean" ? obj.bargeIn : DEFAULT_CONFIG.bargeIn,
    allowedUsers: Array.isArray(obj.allowedUsers) ? obj.allowedUsers.filter((u): u is string => typeof u === "string") : [],
    primaryUser: typeof obj.primaryUser === "string" && obj.primaryUser.trim() ? obj.primaryUser.trim() : undefined,
    allowVoiceSwitch: typeof obj.allowVoiceSwitch === "boolean" ? obj.allowVoiceSwitch : DEFAULT_CONFIG.allowVoiceSwitch,
    wakeWord: typeof obj.wakeWord === "string" && obj.wakeWord.trim() ? obj.wakeWord.trim().toLowerCase() : DEFAULT_CONFIG.wakeWord,
    silenceThresholdMs: typeof obj.silenceThresholdMs === "number" ? obj.silenceThresholdMs : DEFAULT_CONFIG.silenceThresholdMs,
    minAudioMs: typeof obj.minAudioMs === "number" ? obj.minAudioMs : DEFAULT_CONFIG.minAudioMs,
    maxRecordingMs: typeof obj.maxRecordingMs === "number" ? obj.maxRecordingMs : DEFAULT_CONFIG.maxRecordingMs,
    autoJoinChannel: typeof obj.autoJoinChannel === "string" && obj.autoJoinChannel.trim() ? obj.autoJoinChannel.trim() : undefined,
    heartbeatIntervalMs: typeof obj.heartbeatIntervalMs === "number" ? obj.heartbeatIntervalMs : DEFAULT_CONFIG.heartbeatIntervalMs,
    model: typeof obj.model === "string" && obj.model.trim() ? obj.model.trim() : undefined,
    thinkLevel: typeof obj.thinkLevel === "string" && obj.thinkLevel.trim() ? obj.thinkLevel.trim() : undefined,
    deepgram: deepgramRaw ? {
      [REDACTED_SECRET] deepgramRaw.apiKey === "string" ? deepgramRaw.[REDACTED_SECRET],
      sttModel: typeof deepgramRaw.sttModel === "string" && deepgramRaw.sttModel.trim() ? deepgramRaw.sttModel.trim() : "nova-2",
      language: typeof deepgramRaw.language === "string" && deepgramRaw.language.trim() ? deepgramRaw.language.trim() : undefined,
    } : undefined,
  };
}

export function getVadRmsThreshold(sensitivity: "low" | "medium" | "high"): number {
  // RMS thresholds tuned for decoded 16-bit PCM from Discord Opus.
  // Larger threshold = less sensitive.
  switch (sensitivity) {
    case "low":
      return 400;
    case "high":
      return 1200;
    case "medium":
    default:
      return 800;
  }
}


path: src/core-bridge.ts
content: import fs from "node:fs";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";

import type { VoiceCallTtsConfig } from "./config.js";

export type CoreConfig = {
  session?: {
    store?: string;
  };
  messages?: {
    tts?: Record<string, unknown>;
  };
  [key: string]: unknown;
};

type CoreAgentDeps = {
  resolveAgentDir: (cfg: CoreConfig, agentId: string) => string;
  resolveAgentWorkspaceDir: (cfg: CoreConfig, agentId: string) => string;
  resolveAgentIdentity: (
    cfg: CoreConfig,
    agentId: string,
  ) => { name?: string | null } | null | undefined;
  resolveThinkingDefault: (params: {
    cfg: CoreConfig;
    provider?: string;
    model?: string;
  }) => string;
  runEmbeddedPiAgent: (params: {
    sessionId: string;
    sessionKey?: string;
    messageProvider?: string;
    sessionFile: string;
    workspaceDir: string;
    config?: CoreConfig;
    prompt: string;
    provider?: string;
    model?: string;
    thinkLevel?: string;
    verboseLevel?: string;
    timeoutMs: number;
    runId: string;
    lane?: string;
    extraSystemPrompt?: string;
    agentDir?: string;
  }) => Promise<{
    payloads?: Array<{ text?: string; isError?: boolean }>;
    meta?: { aborted?: boolean };
  }>;
  resolveAgentTimeoutMs: (opts: { cfg: CoreConfig }) => number;
  ensureAgentWorkspace: (params?: { dir: string }) => Promise<void>;
  resolveStorePath: (store?: string, opts?: { agentId?: string }) => string;
  loadSessionStore: (storePath: string) => Record<string, unknown>;
  saveSessionStore: (
    storePath: string,
    store: Record<string, unknown>,
  ) => Promise<void>;
  resolveSessionFilePath: (
    sessionId: string,
    entry: unknown,
    opts?: { agentId?: string },
  ) => string;
  DEFAULT_MODEL: string;
  DEFAULT_PROVIDER: string;
};

let coreRootCache: string | null = null;
let coreDepsPromise: Promise<CoreAgentDeps> | null = null;

function findPackageRoot(startDir: string, name: string): string | null {
  let dir = startDir;
  for (;;) {
    const pkgPath = path.join(dir, "package.json");
    try {
      if (fs.existsSync(pkgPath)) {
        const raw = fs.readFileSync(pkgPath, "utf8");
        const pkg = JSON.parse(raw) as { name?: string };
        if (pkg.name === name) return dir;
      }
    } catch {
      // ignore parse errors and keep walking
    }
    const parent = path.dirname(dir);
    if (parent === dir) return null;
    dir = parent;
  }
}

function resolveClawdbotRoot(): string {
  if (coreRootCache) return coreRootCache;
  const override = process.env.CLAWDBOT_ROOT?.trim();
  if (override) {
    coreRootCache = override;
    return override;
  }

  const candidates = new Set<string>();
  if (process.argv[1]) {
    candidates.add(path.dirname(process.argv[1]));
  }
  candidates.add(process.cwd());
  try {
    const urlPath = fileURLToPath(import.meta.url);
    candidates.add(path.dirname(urlPath));
  } catch {
    // ignore
  }

  for (const start of candidates) {
    const found = findPackageRoot(start, "clawdbot");
    if (found) {
      coreRootCache = found;
      return found;
    }
  }

  throw new Error(
    "Unable to resolve Clawdbot root. Set CLAWDBOT_ROOT to the package root.",
  );
}

async function importCoreModule<T>(relativePath: string): Promise<T> {
  const root = resolveClawdbotRoot();
  const distPath = path.join(root, "dist", relativePath);
  if (!fs.existsSync(distPath)) {
    throw new Error(
      `Missing core module at ${distPath}. Run \`pnpm build\` or install the official package.`,
    );
  }
  return (await import(pathToFileURL(distPath).href)) as T;
}

export async function loadCoreAgentDeps(): Promise<CoreAgentDeps> {
  if (coreDepsPromise) return coreDepsPromise;

  coreDepsPromise = (async () => {
    const [
      agentScope,
      defaults,
      identity,
      modelSelection,
      piEmbedded,
      timeout,
      workspace,
      sessions,
    ] = await Promise.all([
      importCoreModule<{
        resolveAgentDir: CoreAgentDeps["resolveAgentDir"];
        resolveAgentWorkspaceDir: CoreAgentDeps["resolveAgentWorkspaceDir"];
      }>("agents/agent-scope.js"),
      importCoreModule<{
        DEFAULT_MODEL: string;
        DEFAULT_PROVIDER: string;
      }>("agents/defaults.js"),
      importCoreModule<{
        resolveAgentIdentity: CoreAgentDeps["resolveAgentIdentity"];
      }>("agents/identity.js"),
      importCoreModule<{
        resolveThinkingDefault: CoreAgentDeps["resolveThinkingDefault"];
      }>("agents/model-selection.js"),
      importCoreModule<{
        runEmbeddedPiAgent: CoreAgentDeps["runEmbeddedPiAgent"];
      }>("agents/pi-embedded.js"),
      importCoreModule<{
        resolveAgentTimeoutMs: CoreAgentDeps["resolveAgentTimeoutMs"];
      }>("agents/timeout.js"),
      importCoreModule<{
        ensureAgentWorkspace: CoreAgentDeps["ensureAgentWorkspace"];
      }>("agents/workspace.js"),
      importCoreModule<{
        resolveStorePath: CoreAgentDeps["resolveStorePath"];
        loadSessionStore: CoreAgentDeps["loadSessionStore"];
        saveSessionStore: CoreAgentDeps["saveSessionStore"];
        resolveSessionFilePath: CoreAgentDeps["resolveSessionFilePath"];
      }>("config/sessions.js"),
    ]);

    return {
      resolveAgentDir: agentScope.resolveAgentDir,
      resolveAgentWorkspaceDir: agentScope.resolveAgentWorkspaceDir,
      resolveAgentIdentity: identity.resolveAgentIdentity,
      resolveThinkingDefault: modelSelection.resolveThinkingDefault,
      runEmbeddedPiAgent: piEmbedded.runEmbeddedPiAgent,
      resolveAgentTimeoutMs: timeout.resolveAgentTimeoutMs,
      ensureAgentWorkspace: workspace.ensureAgentWorkspace,
      resolveStorePath: sessions.resolveStorePath,
      loadSessionStore: sessions.loadSessionStore,
      saveSessionStore: sessions.saveSessionStore,
      resolveSessionFilePath: sessions.resolveSessionFilePath,
      DEFAULT_MODEL: defaults.DEFAULT_MODEL,
      DEFAULT_PROVIDER: defaults.DEFAULT_PROVIDER,
    };
  })();

  return coreDepsPromise;
}


path: src/streaming-stt.ts
content: /**
 * Streaming Speech-to-Text via Deepgram WebSocket
 * 
 * Provides real-time transcription as audio streams in,
 * significantly reducing latency compared to batch transcription.
 */

import { EventEmitter } from "node:events";
import WebSocket from "ws";
import type { DiscordVoiceConfig } from "./config.js";

export interface StreamingSTTEvents {
  transcript: (text: string, isFinal: boolean, confidence?: number) => void;
  error: (error: Error) => void;
  close: () => void;
  ready: () => void;
}

export interface StreamingSTTProvider extends EventEmitter {
  on<K extends keyof StreamingSTTEvents>(event: K, listener: StreamingSTTEvents[K]): this;
  emit<K extends keyof StreamingSTTEvents>(event: K, ...args: Parameters<StreamingSTTEvents[K]>): boolean;
  
  /** Send audio chunk to be transcribed */
  sendAudio(chunk: Buffer): void;
  
  /** Signal end of audio stream */
  finalize(): void;
  
  /** Close the connection */
  close(): void;
  
  /** Check if connection is ready */
  isReady(): boolean;
}

/**
 * Deepgram Streaming STT Provider
 * 
 * Uses WebSocket connection for real-time transcription.
 * Supports interim results for ultra-low latency feedback.
 */
export class DeepgramStreamingSTT extends EventEmitter implements StreamingSTTProvider {
  private ws: WebSocket | null = null;
  private [REDACTED_SECRET];
  private model: string;
  private language?: string;
  private ready = false;
  private closed = false;
  private reconnectAttempts = 0;
  private maxReconnectAttempts = 3;
  private keepAliveInterval: ReturnType<typeof setInterval> | null = null;
  private sampleRate: number;
  private interimResults: boolean;
  private endpointing: number;
  private utteranceEndMs: number;
  
  // Buffer for audio chunks received before connection is ready
  private pendingAudioChunks: Buffer[] = [];
  private maxPendingChunks = 500; // ~5 seconds of audio at 48kHz

  constructor(config: DiscordVoiceConfig, options?: {
    sampleRate?: number;
    interimResults?: boolean;
    endpointing?: number;      // ms of silence to detect end of utterance
    utteranceEndMs?: number;   // ms to wait after utterance end before finalizing
  }) {
    super();
    this.[REDACTED_SECRET] || process.env.DEEPGRAM_API_KEY || "";
    this.model = (config.deepgram?.sttModel || process.env.DEEPGRAM_STT_MODEL || "nova-2");
    this.language = config.deepgram?.language || process.env.DEEPGRAM_LANGUAGE || undefined;
    this.sampleRate = options?.sampleRate ?? 48000;
    this.interimResults = options?.interimResults ?? true;
    this.endpointing = options?.endpointing ?? 300;  // 300ms silence detection
    this.utteranceEndMs = options?.utteranceEndMs ?? 1000;

    if (!this.apiKey) {
      throw new Error("Deepgram API key required for streaming STT");
    }

    this.connect();
  }

  private connect(): void {
    if (this.closed) return;

    // Build Deepgram streaming URL with parameters
    const params = new URLSearchParams({
      model: this.model,
      encoding: "linear16",
      sample_rate: this.sampleRate.toString(),
      channels: "1",
      interim_results: this.interimResults.toString(),
      punctuate: "true",
      endpointing: this.endpointing.toString(),
      utterance_end_ms: this.utteranceEndMs.toString(),
      vad_events: "true",
      smart_format: "true",
    });

    if (this.language) {
      params.set("language", this.language);
    }

    const url = `wss://api.deepgram.com/v1/listen?${params}`;

    this.ws = new WebSocket(url, {
      headers: {
        Authorization: `Token ${this.apiKey}`,
      },
    });

    this.ws.on("open", () => {
      this.ready = true;
      this.reconnectAttempts = 0;
      
      // Flush any pending audio chunks that were buffered during connection
      if (this.pendingAudioChunks.length > 0) {
        console.log(`[streaming-stt] Connection ready, flushing ${this.pendingAudioChunks.length} buffered audio chunks`);
        for (const chunk of this.pendingAudioChunks) {
          this.ws!.send(chunk);
        }
        this.pendingAudioChunks = [];
      }
      
      this.emit("ready");
      
      // Start keep-alive pings every 10 seconds
      this.keepAliveInterval = setInterval(() => {
        if (this.ws?.readyState === WebSocket.OPEN) {
          // Send keep-alive message (empty JSON object)
          this.ws.send(JSON.stringify({ type: "KeepAlive" }));
        }
      }, 10000);
    });

    this.ws.on("message", (data: Buffer | string) => {
      try {
        const msg = JSON.parse(data.toString()) as DeepgramMessage;
        this.handleMessage(msg);
      } catch (err) {
        // Ignore parse errors for non-JSON messages
      }
    });

    this.ws.on("close", (code, reason) => {
      this.ready = false;
      this.clearKeepAlive();
      
      if (!this.closed && this.reconnectAttempts < this.maxReconnectAttempts) {
        this.reconnectAttempts++;
        setTimeout(() => this.connect(), 1000 * this.reconnectAttempts);
      } else {
        this.emit("close");
      }
    });

    this.ws.on("error", (err) => {
      this.emit("error", err);
    });
  }

  private handleMessage(msg: DeepgramMessage): void {
    if (msg.type === "Results" && msg.channel?.alternatives?.[0]) {
      const alt = msg.channel.alternatives[0];
      const text = alt.transcript?.trim();
      
      if (text && text.length > 0) {
        const isFinal = msg.is_final ?? false;
        this.emit("transcript", text, isFinal, alt.confidence);
      }
    } else if (msg.type === "UtteranceEnd") {
      // Utterance ended - can be used to trigger processing
      // The final transcript should have already been emitted
    } else if (msg.type === "SpeechStarted") {
      // Speech detected - could use for barge-in detection
    }
  }

  private clearKeepAlive(): void {
    if (this.keepAliveInterval) {
      clearInterval(this.keepAliveInterval);
      this.keepAliveInterval = null;
    }
  }

  sendAudio(chunk: Buffer): void {
    if (this.ws?.readyState === WebSocket.OPEN) {
      this.ws.send(chunk);
    } else if (!this.closed) {
      // Buffer audio while connection is establishing
      this.pendingAudioChunks.push(chunk);
      // Prevent unlimited buffering
      if (this.pendingAudioChunks.length > this.maxPendingChunks) {
        this.pendingAudioChunks.shift(); // Drop oldest chunk
      }
    }
  }

  finalize(): void {
    if (this.ws?.readyState === WebSocket.OPEN) {
      // Send close stream message to get final results
      this.ws.send(JSON.stringify({ type: "CloseStream" }));
    }
  }

  close(): void {
    this.closed = true;
    this.clearKeepAlive();
    this.pendingAudioChunks = [];
    if (this.ws) {
      this.ws.close();
      this.ws = null;
    }
    this.ready = false;
  }

  isReady(): boolean {
    return this.ready && this.ws?.readyState === WebSocket.OPEN;
  }

  /**
   * Wait for the connection to be ready (or fail)
   * Returns true if ready, false if failed/closed
   */
  waitForReady(timeoutMs = 5000): Promise<boolean> {
    if (this.isReady()) return Promise.resolve(true);
    if (this.closed) return Promise.resolve(false);

    return new Promise((resolve) => {
      const timeout = setTimeout(() => {
        this.off("ready", onReady);
        this.off("close", onClose);
        resolve(false);
      }, timeoutMs);

      const onReady = () => {
        clearTimeout(timeout);
        this.off("close", onClose);
        resolve(true);
      };

      const onClose = () => {
        clearTimeout(timeout);
        this.off("ready", onReady);
        resolve(false);
      };

      this.once("ready", onReady);
      this.once("close", onClose);
    });
  }
}

// Deepgram message types
interface DeepgramMessage {
  type: "Results" | "Metadata" | "UtteranceEnd" | "SpeechStarted" | "Error";
  is_final?: boolean;
  speech_final?: boolean;
  channel?: {
    alternatives?: Array<{
      transcript?: string;
      confidence?: number;
      words?: Array<{
        word: string;
        start: number;
        end: number;
        confidence: number;
      }>;
    }>;
  };
  metadata?: {
    request_id?: string;
    model_info?: {
      name: string;
      version: string;
    };
  };
}

/**
 * Streaming STT Session Manager
 * 
 * Manages per-user streaming STT sessions with automatic
 * lifecycle handling (create on speech start, destroy on silence).
 */
export class StreamingSTTManager {
  private config: DiscordVoiceConfig;
  private sessions: Map<string, DeepgramStreamingSTT> = new Map();
  private pendingTranscripts: Map<string, string> = new Map();
  
  constructor(config: DiscordVoiceConfig) {
    this.config = config;
  }

  /**
   * Get or create a streaming session for a user
   */
  getOrCreateSession(
    userId: string,
    onTranscript: (text: string, isFinal: boolean) => void
  ): DeepgramStreamingSTT {
    let session = this.sessions.get(userId);
    
    if (!session || !session.isReady()) {
      // Clean up old session if exists
      if (session) {
        session.close();
      }

      session = new DeepgramStreamingSTT(this.config, {
        sampleRate: 48000,
        interimResults: true,
        endpointing: 300,
        utteranceEndMs: 1000,
      });

      // Track partial transcripts for this user
      this.pendingTranscripts.set(userId, "");

      session.on("transcript", (text, isFinal, confidence) => {
        if (isFinal) {
          // Accumulate final transcripts
          const pending = this.pendingTranscripts.get(userId) || "";
          const fullText = pending ? `${pending} ${text}` : text;
          this.pendingTranscripts.set(userId, fullText);
        }
        onTranscript(text, isFinal);
      });

      session.on("close", () => {
        this.sessions.delete(userId);
      });

      session.on("error", (err) => {
        console.error(`[streaming-stt] Error for user ${userId}:`, err.message);
      });

      this.sessions.set(userId, session);
    }

    return session;
  }

  /**
   * Send audio to user's session
   * Audio is buffered if connection is still establishing
   */
  sendAudio(userId: string, chunk: Buffer): void {
    const session = this.sessions.get(userId);
    if (session) {
      // sendAudio now handles buffering internally if not ready
      session.sendAudio(chunk);
    }
  }

  /**
   * Finalize and get accumulated transcript for user
   */
  finalizeSession(userId: string): string {
    const session = this.sessions.get(userId);
    if (session) {
      session.finalize();
    }
    
    const transcript = this.pendingTranscripts.get(userId) || "";
    this.pendingTranscripts.delete(userId);
    return transcript;
  }

  /**
   * Close a user's session
   */
  closeSession(userId: string): void {
    const session = this.sessions.get(userId);
    if (session) {
      session.close();
      this.sessions.delete(userId);
    }
    this.pendingTranscripts.delete(userId);
  }

  /**
   * Close all sessions
   */
  closeAll(): void {
    for (const [userId, session] of this.sessions) {
      session.close();
    }
    this.sessions.clear();
    this.pendingTranscripts.clear();
  }
}

/**
 * Create streaming STT provider based on config
 */
export function createStreamingSTTProvider(config: DiscordVoiceConfig): StreamingSTTManager | null {
  if (!config.streamingSTT) {
    return null;
  }
  
  return new StreamingSTTManager(config);
}


path: src/streaming-tts.ts
content: /**
 * Low-latency Text-to-Speech using Deepgram Aura.
 *
 * Note: Deepgram's TTS WebSocket endpoint streams only linear16/mulaw/alaw.
 * For best Discord compatibility we stream Ogg/Opus over HTTP from /v1/speak.
 */

import { Readable } from "node:stream";
import type { DiscordVoiceConfig } from "./config.js";

export interface StreamingTTSResult {
  stream: Readable;
  format: "opus";
  sampleRate: number;
}

export interface StreamingTTSProvider {
  synthesizeStream(text: string): Promise<StreamingTTSResult>;
}

function mustGetDeepgramKey(cfg: DiscordVoiceConfig): string {
  const key = cfg.deepgram?.apiKey || process.env.DEEPGRAM_API_KEY || "";
  if (!key) throw new Error("Deepgram API key required for TTS");
  return key;
}

function mustGetVoice(cfg: DiscordVoiceConfig): string {
  return cfg.ttsVoice || "aura-2-thalia-en";
}

export class DeepgramHttpStreamingTTS implements StreamingTTSProvider {
  private [REDACTED_SECRET];
  private voiceModel: string;
  private sampleRate: number;

  constructor(cfg: DiscordVoiceConfig, opts?: { sampleRate?: number }) {
    this.[REDACTED_SECRET]);
    this.voiceModel = mustGetVoice(cfg);
    this.sampleRate = opts?.sampleRate ?? 48000;
  }

  async synthesizeStream(text: string): Promise<StreamingTTSResult> {
    const controller = new AbortController();
    const timeout = setTimeout(() => controller.abort(), 25_000);

    try {
      const url = new URL("https://api.deepgram.com/v1/speak");
      url.searchParams.set("model", this.voiceModel);
      url.searchParams.set("encoding", "opus");
      url.searchParams.set("container", "ogg");
      url.searchParams.set("sample_rate", String(this.sampleRate));

      const resp = await fetch(url.toString(), {
        method: "POST",
        headers: {
          Authorization: `Token ${this.apiKey}`,
          "Content-Type": "application/json",
        },
        body: JSON.stringify({ text }),
        signal: controller.signal,
      });

      if (!resp.ok) {
        const errText = await resp.text().catch(() => "");
        throw new Error(`Deepgram TTS error: ${resp.status} ${resp.statusText} ${errText}`);
      }

      if (!resp.body) {
        throw new Error("Deepgram TTS response missing body stream");
      }

      // Node 18+ provides Readable.fromWeb for streaming WebResponse bodies.
      const nodeStream = Readable.fromWeb(resp.body as any);

      return {
        stream: nodeStream,
        format: "opus",
        sampleRate: this.sampleRate,
      };
    } finally {
      clearTimeout(timeout);
    }
  }
}

export function createStreamingTTSProvider(cfg: DiscordVoiceConfig): StreamingTTSProvider | null {
  if (!cfg.streamingTTS) return null;
  return new DeepgramHttpStreamingTTS(cfg, { sampleRate: 48000 });
}


path: src/stt.ts
content: /**
 * Buffered Speech-to-Text using Deepgram (fallback path).
 */

import type { DiscordVoiceConfig } from "./config.js";

export interface STTResult {
  text: string;
  confidence?: number;
}

export interface STTProvider {
  transcribe(audioBuffer: Buffer, sampleRate: number): Promise<STTResult>;
}

export class DeepgramSTT implements STTProvider {
  private [REDACTED_SECRET];
  private model: string;
  private language?: string;

  constructor(cfg: DiscordVoiceConfig) {
    this.[REDACTED_SECRET] || process.env.DEEPGRAM_API_KEY || "";
    this.model = cfg.deepgram?.sttModel || process.env.DEEPGRAM_STT_MODEL || "nova-2";
    this.language = cfg.deepgram?.language || process.env.DEEPGRAM_LANGUAGE || undefined;
    if (!this.apiKey) throw new Error("Deepgram API key required for STT");
  }

  async transcribe(audioBuffer: Buffer, sampleRate: number): Promise<STTResult> {
    const url = new URL("https://api.deepgram.com/v1/listen");
    url.searchParams.set("model", this.model);
    url.searchParams.set("encoding", "linear16");
    url.searchParams.set("sample_rate", String(sampleRate));
    url.searchParams.set("channels", "1");
    url.searchParams.set("punctuate", "true");
    url.searchParams.set("smart_format", "true");
    if (this.language) url.searchParams.set("language", this.language);

    const resp = await fetch(url.toString(), {
      method: "POST",
      headers: {
        Authorization: `Token ${this.apiKey}`,
        "Content-Type": "application/octet-stream",
      },
      body: audioBuffer,
    });

    if (!resp.ok) {
      const errText = await resp.text().catch(() => "");
      throw new Error(`Deepgram STT error: ${resp.status} ${resp.statusText} ${errText}`);
    }

    const result = (await resp.json()) as any;
    const alt = result?.results?.channels?.[0]?.alternatives?.[0];
    return {
      text: (alt?.transcript || "").trim(),
      confidence: alt?.confidence,
    };
  }
}

export function createSTTProvider(cfg: DiscordVoiceConfig): STTProvider {
  return new DeepgramSTT(cfg);
}


path: src/tts.ts
content: /**
 * Buffered Text-to-Speech using Deepgram Aura (fallback path).
 */

import type { DiscordVoiceConfig } from "./config.js";

export interface TTSResult {
  audioBuffer: Buffer;
  format: "opus";
  sampleRate: number;
}

export interface TTSProvider {
  synthesize(text: string): Promise<TTSResult>;
}

export class DeepgramTTS implements TTSProvider {
  private [REDACTED_SECRET];
  private voiceModel: string;
  private sampleRate: number;

  constructor(cfg: DiscordVoiceConfig, opts?: { sampleRate?: number }) {
    this.[REDACTED_SECRET] || process.env.DEEPGRAM_API_KEY || "";
    if (!this.apiKey) throw new Error("Deepgram API key required for TTS");
    this.voiceModel = cfg.ttsVoice || "aura-2-thalia-en";
    this.sampleRate = opts?.sampleRate ?? 48000;
  }

  async synthesize(text: string): Promise<TTSResult> {
    const url = new URL("https://api.deepgram.com/v1/speak");
    url.searchParams.set("model", this.voiceModel);
    url.searchParams.set("encoding", "opus");
    url.searchParams.set("container", "ogg");
    url.searchParams.set("sample_rate", String(this.sampleRate));

    const resp = await fetch(url.toString(), {
      method: "POST",
      headers: {
        Authorization: `Token ${this.apiKey}`,
        "Content-Type": "application/json",
      },
      body: JSON.stringify({ text }),
    });

    if (!resp.ok) {
      const errText = await resp.text().catch(() => "");
      throw new Error(`Deepgram TTS error: ${resp.status} ${resp.statusText} ${errText}`);
    }

    const arr = await resp.arrayBuffer();
    return {
      audioBuffer: Buffer.from(arr),
      format: "opus",
      sampleRate: this.sampleRate,
    };
  }
}

export function createTTSProvider(cfg: DiscordVoiceConfig): TTSProvider {
  return new DeepgramTTS(cfg, { sampleRate: 48000 });
}


path: src/voice-connection.ts
content: /**
 * Discord Voice Connection Manager
 * Handles joining, leaving, listening, and speaking in voice channels
 * 
 * Features:
 * - Barge-in: Stops speaking when user starts talking
 * - Auto-reconnect heartbeat: Keeps connection alive
 * - Streaming STT: Real-time transcription with Deepgram
 */

import {
  joinVoiceChannel,
  createAudioPlayer,
  createAudioResource,
  AudioPlayerStatus,
  VoiceConnectionStatus,
  entersState,
  getVoiceConnection,
  EndBehaviorType,
  StreamType,
  type VoiceConnection,
  type AudioPlayer,
  type AudioReceiveStream,
} from "@discordjs/voice";
import type {
  VoiceChannel,
  StageChannel,
  GuildMember,
  VoiceBasedChannel,
} from "discord.js";
import { Readable, PassThrough } from "stream";
import { pipeline } from "stream/promises";
import * as prism from "prism-media";

import type { DiscordVoiceConfig } from "./config.js";
import { getVadRmsThreshold } from "./config.js";

import { createSTTProvider, type STTProvider } from "./stt.js";
import { createStreamingSTTProvider, StreamingSTTManager } from "./streaming-stt.js";
import { createTTSProvider, type TTSProvider } from "./tts.js";
import { createStreamingTTSProvider, type StreamingTTSProvider } from "./streaming-tts.js";

/** Minimal logger interface used by this plugin */
export interface Logger {
  info: (msg: string) => void;
  warn: (msg: string) => void;
  error: (msg: string) => void;
  debug?: (msg: string) => void;
}


/**
 * Get RMS threshold based on VAD sensitivity
 * Higher = less sensitive (filters more noise)
 */
 interface UserAudioState {
  chunks: Buffer[];
  lastActivityMs: number;
  isRecording: boolean;
  silenceTimer?: ReturnType<typeof setTimeout>;
  opusStream?: AudioReceiveStream;
  decoder?: prism.opus.Decoder;
}

export interface VoiceSession {
  guildId: string;
  channelId: string;
  channelName?: string;
  channelRef?: VoiceBasedChannel;
  primaryUserId?: string;
  activeSpeakerId?: string;
  connection: VoiceConnection;
  player: AudioPlayer;
  userAudioStates: Map<string, UserAudioState>;
  speaking: boolean;
  processing: boolean;           // Lock to prevent concurrent processing
  lastSpokeAt?: number;          // Timestamp when bot finished speaking (for cooldown)
  startedSpeakingAt?: number;    // Timestamp when bot started speaking (for echo suppression)
  thinkingPlayer?: AudioPlayer;  // Separate player for thinking sound
  heartbeatInterval?: ReturnType<typeof setInterval>;
  lastHeartbeat?: number;
  reconnecting?: boolean;
}

export class VoiceConnectionManager {
  private sessions: Map<string, VoiceSession> = new Map();
  private config: DiscordVoiceConfig;
  private sttProvider: STTProvider | null = null;
  private streamingSTT: StreamingSTTManager | null = null;
  private ttsProvider: TTSProvider | null = null;
  private streamingTTS: StreamingTTSProvider | null = null;
  private logger: Logger;
  private onTranscript: (userId: string, guildId: string, channelId: string, text: string) => Promise<string>;
  private botUserId: string | null = null;

  // Heartbeat configuration (can be overridden via config.heartbeatIntervalMs)
  private readonly DEFAULT_HEARTBEAT_INTERVAL_MS = 30_000;  // 30 seconds
  private readonly HEARTBEAT_TIMEOUT_MS = 60_000;   // 60 seconds before reconnect
  private readonly MAX_RECONNECT_ATTEMPTS = 3;
  
  private get HEARTBEAT_INTERVAL_MS(): number {
    return this.config.heartbeatIntervalMs ?? this.DEFAULT_HEARTBEAT_INTERVAL_MS;
  }

  constructor(
    config: DiscordVoiceConfig,
    logger: Logger,
    onTranscript: (userId: string, guildId: string, channelId: string, text: string) => Promise<string>,
    botUserId?: string
  ) {
    this.config = config;
    this.logger = logger;
    this.onTranscript = onTranscript;
    this.botUserId = botUserId || null;
  }
  
  /**
   * Set the bot's user ID (for filtering out echo)
   */
  setBotUserId(userId: string): void {
    this.botUserId = userId;
    this.logger.info(`[discord-voice] Bot user ID set to ${userId}`);
  }

  /**
   * Initialize providers lazily
   */
  private ensureProviders(): void {
    if (!this.sttProvider) {
      this.sttProvider = createSTTProvider(this.config);
    }
    if (!this.ttsProvider) {
      this.ttsProvider = createTTSProvider(this.config);
    }
    // Initialize streaming TTS (always, for lower latency)
    if (!this.streamingTTS) {
      this.streamingTTS = createStreamingTTSProvider(this.config);
    }
    // Initialize streaming STT if using Deepgram with streaming enabled
    if (!this.streamingSTT && this.config.streamingSTT) {
      this.streamingSTT = createStreamingSTTProvider(this.config);
    }
  }

  /**
   * Join a voice channel
   */
  async join(channel: VoiceBasedChannel): Promise<VoiceSession> {
    const existingSession = this.sessions.get(channel.guildId);
    if (existingSession) {
      if (existingSession.channelId === channel.id) {
        return existingSession;
      }
      // Leave current channel first
      await this.leave(channel.guildId);
    }

    this.ensureProviders();

    const connection = joinVoiceChannel({
      channelId: channel.id,
      guildId: channel.guildId,
      adapterCreator: channel.guild.voiceAdapterCreator,
      selfDeaf: false, // We need to hear users
      selfMute: false,
    });

    const player = createAudioPlayer();
    connection.subscribe(player);

    const session: VoiceSession = {
      guildId: channel.guildId,
      channelId: channel.id,
      channelName: channel.name,
      channelRef: channel,
      activeSpeakerId: undefined,
      connection,
      player,
      userAudioStates: new Map(),
      speaking: false,
      processing: false,
      lastHeartbeat: Date.now(),
    };

    this.sessions.set(channel.guildId, session);

    // Resolve primary speaker (if configured)
    this.resolvePrimaryUser(session);

    // Wait for the connection to be ready
    try {
      await entersState(connection, VoiceConnectionStatus.Ready, 20_000);
      this.logger.info(`[discord-voice] Joined voice channel ${channel.name} in ${channel.guild.name}`);
    } catch (error) {
      connection.destroy();
      this.sessions.delete(channel.guildId);
      throw new Error(`Failed to join voice channel: ${error}`);
    }

    // Start listening to users
    this.startListening(session);

    // Start heartbeat for connection health monitoring
    this.startHeartbeat(session);

    // Handle connection state changes
    this.setupConnectionHandlers(session, channel);

    return session;
  }

  /**
   * Setup connection event handlers for auto-reconnect
   */
  private setupConnectionHandlers(session: VoiceSession, channel: VoiceBasedChannel): void {
    const connection = session.connection;

    connection.on(VoiceConnectionStatus.Disconnected, async () => {
      if (session.reconnecting) return;
      
      this.logger.warn(`[discord-voice] Disconnected from voice channel in ${channel.guild.name}`);
      
      try {
        // Try to reconnect within 5 seconds
        await Promise.race([
          entersState(connection, VoiceConnectionStatus.Signalling, 5_000),
          entersState(connection, VoiceConnectionStatus.Connecting, 5_000),
        ]);
        this.logger.info(`[discord-voice] Reconnecting to voice channel...`);
      } catch {
        // Connection is not recovering, attempt manual reconnect
        await this.attemptReconnect(session, channel);
      }
    });

    connection.on(VoiceConnectionStatus.Ready, () => {
      session.lastHeartbeat = Date.now();
      session.reconnecting = false;
      this.logger.info(`[discord-voice] Connection ready for ${channel.name}`);
    });

    connection.on("error", (error) => {
      this.logger.error(`[discord-voice] Connection error: ${error.message}`);
    });
  }

  /**
   * Attempt to reconnect to voice channel
   */
  private async attemptReconnect(session: VoiceSession, channel: VoiceBasedChannel, attempt = 1): Promise<void> {
    if (attempt > this.MAX_RECONNECT_ATTEMPTS) {
      this.logger.error(`[discord-voice] Max reconnection attempts reached, giving up`);
      await this.leave(session.guildId);
      return;
    }

    session.reconnecting = true;
    this.logger.info(`[discord-voice] Reconnection attempt ${attempt}/${this.MAX_RECONNECT_ATTEMPTS}`);

    try {
      // Destroy old connection
      session.connection.destroy();

      // Wait before reconnecting (exponential backoff)
      await new Promise(resolve => setTimeout(resolve, 1000 * attempt));

      // Create new connection
      const newConnection = joinVoiceChannel({
        channelId: channel.id,
        guildId: channel.guildId,
        adapterCreator: channel.guild.voiceAdapterCreator,
        selfDeaf: false,
        selfMute: false,
      });

      const newPlayer = createAudioPlayer();
      newConnection.subscribe(newPlayer);

      // Update session
      session.connection = newConnection;
      session.player = newPlayer;

      // Wait for ready
      await entersState(newConnection, VoiceConnectionStatus.Ready, 20_000);
      
      session.reconnecting = false;
      session.lastHeartbeat = Date.now();
      
      // Restart listening
      this.startListening(session);
      
      // Setup handlers for new connection
      this.setupConnectionHandlers(session, channel);
      
      this.logger.info(`[discord-voice] Reconnected successfully`);
    } catch (error) {
      this.logger.error(`[discord-voice] Reconnection failed: ${error instanceof Error ? error.message : String(error)}`);
      await this.attemptReconnect(session, channel, attempt + 1);
    }
  }

  /**
   * Start heartbeat monitoring for session
   */
  private startHeartbeat(session: VoiceSession): void {
    // Clear any existing heartbeat
    if (session.heartbeatInterval) {
      clearInterval(session.heartbeatInterval);
    }

    session.heartbeatInterval = setInterval(() => {
      const now = Date.now();
      const connectionState = session.connection.state.status;
      
      // Update heartbeat if connection is healthy
      if (connectionState === VoiceConnectionStatus.Ready) {
        session.lastHeartbeat = now;
        this.logger.debug?.(`[discord-voice] Heartbeat OK for guild ${session.guildId}`);
      } else if (session.lastHeartbeat && (now - session.lastHeartbeat > this.HEARTBEAT_TIMEOUT_MS)) {
        // Connection has been unhealthy for too long
        this.logger.warn(`[discord-voice] Heartbeat timeout, connection state: ${connectionState}`);
        
        // Don't attempt reconnect if already doing so
        if (!session.reconnecting) {
          // Trigger reconnection by destroying and rejoining
          this.logger.info(`[discord-voice] Triggering reconnection due to heartbeat timeout`);
          session.connection.destroy();
        }
      }
    }, this.HEARTBEAT_INTERVAL_MS);
  }

  /**
   * Leave a voice channel
   */
  async leave(guildId: string): Promise<boolean> {
    const session = this.sessions.get(guildId);
    if (!session) {
      return false;
    }

    // Clear heartbeat
    if (session.heartbeatInterval) {
      clearInterval(session.heartbeatInterval);
    }

    // Clear all user timers and streams
    for (const state of session.userAudioStates.values()) {
      if (state.silenceTimer) {
        clearTimeout(state.silenceTimer);
      }
      if (state.opusStream) {
        state.opusStream.destroy();
      }
      if (state.decoder) {
        state.decoder.destroy();
      }
    }

    // Close streaming STT sessions
    if (this.streamingSTT) {
      for (const userId of session.userAudioStates.keys()) {
        this.streamingSTT.closeSession(userId);
      }
    }

    session.connection.destroy();
    this.sessions.delete(guildId);
    this.logger.info(`[discord-voice] Left voice channel in guild ${guildId}`);
    return true;
  }

  /**
   * Start listening to voice in the channel
   */
  private startListening(session: VoiceSession): void {
    const receiver = session.connection.receiver;

    receiver.speaking.on("start", (userId: string) => {
      // ═══════════════════════════════════════════════════════════════
      // ECHO FILTER: Ignore speech events from the bot itself
      // This is the primary defense against echo-triggered barge-in
      // ═══════════════════════════════════════════════════════════════
      if (this.botUserId && userId === this.botUserId) {
        this.logger.debug?.(`[discord-voice] Ignoring speech from bot itself (echo filter)`);
        return;
      }
      
      if (!this.isUserAllowed(session, userId)) {
        return;
      }

      // Ignore audio during cooldown period (prevents residual echo)
      const SPEAK_COOLDOWN_MS = 500;
      if (session.lastSpokeAt && (Date.now() - session.lastSpokeAt) < SPEAK_COOLDOWN_MS) {
        this.logger.debug?.(`[discord-voice] Ignoring speech during cooldown (likely residual echo)`);
        return;
      }

      this.logger.debug?.(`[discord-voice] User ${userId} started speaking`);
      
      // ═══════════════════════════════════════════════════════════════
      // BARGE-IN: If we're speaking and a REAL user starts talking, stop
      // Now that we filter out bot's own userId, we can safely do barge-in
      // ═══════════════════════════════════════════════════════════════
      if (session.speaking) {
        if (this.config.bargeIn) {
          this.logger.info(`[discord-voice] Barge-in detected from user ${userId}! Stopping speech.`);
          this.stopSpeaking(session);
          session.lastSpokeAt = Date.now();
        }
        // Clear streaming transcripts and wait for next speech event
        if (this.streamingSTT) {
          this.streamingSTT.closeSession(userId);
        }
        return;
      }
      
      if (session.processing) {
        // While processing a request, don't start new recordings
        if (this.streamingSTT) {
          this.streamingSTT.closeSession(userId);
        }
        this.logger.debug?.(`[discord-voice] Ignoring speech while processing`);
        return;
      }

      let state = session.userAudioStates.get(userId);
      if (!state) {
        state = {
          chunks: [],
          lastActivityMs: Date.now(),
          isRecording: false,
        };
        session.userAudioStates.set(userId, state);
      }

      // Clear any existing silence timer
      if (state.silenceTimer) {
        clearTimeout(state.silenceTimer);
        state.silenceTimer = undefined;
      }

      if (!state.isRecording) {
        state.isRecording = true;
        state.chunks = [];
        this.startRecording(session, userId);
      }

      state.lastActivityMs = Date.now();
    });

    receiver.speaking.on("end", (userId: string) => {
      if (!this.isUserAllowed(session, userId)) {
        return;
      }

      this.logger.debug?.(`[discord-voice] User ${userId} stopped speaking`);
      
      const state = session.userAudioStates.get(userId);
      if (!state || !state.isRecording) {
        return;
      }

      state.lastActivityMs = Date.now();

      // Set silence timer to process the recording
      state.silenceTimer = setTimeout(async () => {
        if (state.isRecording && state.chunks.length > 0) {
          state.isRecording = false;
          
          // Clean up streams
          if (state.opusStream) {
            state.opusStream.destroy();
            state.opusStream = undefined;
          }
          if (state.decoder) {
            state.decoder.destroy();
            state.decoder = undefined;
          }
          
          await this.processRecording(session, userId, state.chunks);
          state.chunks = [];
        }
      }, this.config.silenceThresholdMs);
    });
  }

  /**
   * Stop any current speech output (for barge-in)
   */
  private stopSpeaking(session: VoiceSession): void {
    // Stop main player
    if (session.player.state.status !== AudioPlayerStatus.Idle) {
      session.player.stop(true);
    }
    
    // Stop thinking player if active
    if (session.thinkingPlayer && session.thinkingPlayer.state.status !== AudioPlayerStatus.Idle) {
      session.thinkingPlayer.stop(true);
      session.thinkingPlayer.removeAllListeners();
      session.thinkingPlayer = undefined;
    }

    session.speaking = false;
  }

  /**
   * Start recording audio from a user
   */
  private startRecording(session: VoiceSession, userId: string): void {
    const state = session.userAudioStates.get(userId);
    if (!state) return;

    const opusStream = session.connection.receiver.subscribe(userId, {
      end: {
        behavior: EndBehaviorType.AfterSilence,
        duration: this.config.silenceThresholdMs,
      },
    });

    state.opusStream = opusStream;

    // Decode Opus to PCM
    const decoder = new prism.opus.Decoder({
      rate: 48000,
      channels: 1,
      frameSize: 960,
    });

    state.decoder = decoder;
    opusStream.pipe(decoder);

    // If streaming STT is available and enabled, use it
    const useStreaming = this.streamingSTT && this.config.streamingSTT;
    
    if (useStreaming && this.streamingSTT) {
      // Create streaming session for this user
      const streamingSession = this.streamingSTT.getOrCreateSession(userId, (text, isFinal) => {
        if (isFinal) {
          this.logger.debug?.(`[discord-voice] Streaming transcript (final): "${text}"`);
        } else {
          this.logger.debug?.(`[discord-voice] Streaming transcript (interim): "${text}"`);
        }
      });

      decoder.on("data", (chunk: Buffer) => {
        if (state.isRecording) {
          // Send to streaming STT
          this.streamingSTT?.sendAudio(userId, chunk);
          
          // Also buffer for fallback/debugging
          state.chunks.push(chunk);
          state.lastActivityMs = Date.now();

          // Check max recording length
          const totalSize = state.chunks.reduce((sum, c) => sum + c.length, 0);
          const durationMs = (totalSize / 2) / 48; // 16-bit samples at 48kHz
          if (durationMs >= this.config.maxRecordingMs) {
            this.logger.debug?.(`[discord-voice] Max recording length reached for user ${userId}`);
            state.isRecording = false;
            this.processRecording(session, userId, state.chunks);
            state.chunks = [];
          }
        }
      });
    } else {
      // Batch mode - just buffer audio
      decoder.on("data", (chunk: Buffer) => {
        if (state.isRecording) {
          state.chunks.push(chunk);
          state.lastActivityMs = Date.now();

          // Check max recording length
          const totalSize = state.chunks.reduce((sum, c) => sum + c.length, 0);
          const durationMs = (totalSize / 2) / 48; // 16-bit samples at 48kHz
          if (durationMs >= this.config.maxRecordingMs) {
            this.logger.debug?.(`[discord-voice] Max recording length reached for user ${userId}`);
            state.isRecording = false;
            this.processRecording(session, userId, state.chunks);
            state.chunks = [];
          }
        }
      });
    }

    decoder.on("end", () => {
      this.logger.debug?.(`[discord-voice] Decoder stream ended for user ${userId}`);
    });

    decoder.on("error", (error: Error) => {
      this.logger.error(`[discord-voice] Decoder error for user ${userId}: ${error.message}`);
    });
  }

  /**
   * Process recorded audio through STT and get response
   */
  private async processRecording(session: VoiceSession, userId: string, chunks: Buffer[]): Promise<void> {
    if (!this.sttProvider || !this.ttsProvider) {
      return;
    }

    // Skip if already speaking (prevents overlapping responses)
    if (session.speaking) {
      this.logger.debug?.(`[discord-voice] Skipping processing - already speaking`);
      return;
    }

    // Skip if already processing another request (prevents duplicate responses)
    if (session.processing) {
      this.logger.debug?.(`[discord-voice] Skipping processing - already processing another request`);
      return;
    }

    // Cooldown after speaking to prevent echo/accidental triggers (500ms)
    const SPEAK_COOLDOWN_MS = 500;
    if (session.lastSpokeAt && (Date.now() - session.lastSpokeAt) < SPEAK_COOLDOWN_MS) {
      this.logger.debug?.(`[discord-voice] Skipping processing - in cooldown period after speaking`);
      return;
    }

    const audioBuffer = Buffer.concat(chunks);
    
    // Skip very short recordings (likely noise) - check BEFORE setting processing lock
    const durationMs = (audioBuffer.length / 2) / 48; // 16-bit samples at 48kHz
    if (durationMs < this.config.minAudioMs) {
      this.logger.debug?.(`[discord-voice] Skipping short recording (${Math.round(durationMs)}ms < ${this.config.minAudioMs}ms) for user ${userId}`);
      return;
    }

    // Calculate RMS amplitude to filter out quiet sounds (keystrokes, background noise)
    const rms = this.calculateRMS(audioBuffer);
    const minRMS = getVadRmsThreshold(this.config.vadSensitivity);
    if (rms < minRMS) {
      this.logger.debug?.(`[discord-voice] Skipping quiet audio (RMS ${Math.round(rms)} < ${minRMS}) for user ${userId}`);
      return;
    }

    // Set processing lock AFTER passing all filters
    session.processing = true;

    this.logger.info(`[discord-voice] Processing ${Math.round(durationMs)}ms of audio (RMS: ${Math.round(rms)}) from user ${userId}`);

    try {
      let transcribedText: string;

      // Check if we have streaming transcript available
      if (this.streamingSTT && this.config.streamingSTT) {
        // Get accumulated transcript from streaming session
        transcribedText = this.streamingSTT.finalizeSession(userId);
        
        // Fallback to batch if streaming didn't capture anything
        if (!transcribedText || transcribedText.trim().length === 0) {
          this.logger.debug?.(`[discord-voice] Streaming empty, falling back to batch STT`);
          const sttResult = await this.sttProvider.transcribe(audioBuffer, 48000);
          transcribedText = sttResult.text;
        }
      } else {
        // Batch transcription
        const sttResult = await this.sttProvider.transcribe(audioBuffer, 48000);
        transcribedText = sttResult.text;
      }
      
      if (!transcribedText || transcribedText.trim().length === 0) {
        this.logger.debug?.(`[discord-voice] Empty transcription for user ${userId}`);
        session.processing = false;
        return;
      }

      this.logger.info(`[discord-voice] Transcribed: "${transcribedText}"`);

      // Voice command: primary speaker can switch who is allowed to talk
      const cmd = this.tryHandleSpeakerCommand(session, userId, transcribedText);
      if (cmd.handled) {
        session.processing = false;
        if (cmd.reply && cmd.reply.trim()) {
          session.connection.subscribe(session.player);
          await this.speak(session.guildId, cmd.reply);
        }
        return;
      }

      // Play looping thinking sound while processing
      const stopThinking = await this.startThinkingLoop(session);

      let response: string;
      try {
        // Get response from agent
        response = await this.onTranscript(userId, session.guildId, session.channelId, transcribedText);
      } finally {
        // Always stop thinking sound, even on error
        stopThinking();
        await new Promise(resolve => setTimeout(resolve, 100));
      }
      
      if (!response || response.trim().length === 0) {
        session.processing = false;
        return;
      }

      // Ensure main player is subscribed before speaking
      session.connection.subscribe(session.player);
      
      // Synthesize and play response
      await this.speak(session.guildId, response);
    } catch (error) {
      this.logger.error(`[discord-voice] Error processing audio: ${error instanceof Error ? error.message : String(error)}`);
    } finally {
      session.processing = false;
    }
  }

  /**
   * Speak text in the voice channel
   */
  async speak(guildId: string, text: string): Promise<void> {
    const session = this.sessions.get(guildId);
    if (!session) {
      throw new Error("Not connected to voice channel");
    }

    this.ensureProviders();

    if (!this.streamingTTS && !this.ttsProvider) {
      throw new Error("TTS provider not initialized");
    }

    session.speaking = true;
    session.startedSpeakingAt = Date.now();

    try {
      this.logger.info(`[discord-voice] Speaking: "${text.substring(0, 50)}${text.length > 50 ? "..." : ""}"`);
      
      let resource;

      // Try streaming TTS first (lower latency)
      if (this.streamingTTS) {
        try {
          const streamResult = await this.streamingTTS.synthesizeStream(text);
          
          // Create audio resource from stream
          if (streamResult.format === "opus") {
            resource = createAudioResource(streamResult.stream, {
              inputType: StreamType.OggOpus,
            });
          } else {
            // For mp3, the audio player will transcode
            resource = createAudioResource(streamResult.stream);
          }
          
          this.logger.debug?.(`[discord-voice] Using streaming TTS`);
        } catch (streamError) {
          this.logger.warn(`[discord-voice] Streaming TTS failed, falling back to buffered: ${streamError instanceof Error ? streamError.message : String(streamError)}`);
          // Fall through to buffered TTS
        }
      }

      // Fallback to buffered TTS
      if (!resource && this.ttsProvider) {
        const ttsResult = await this.ttsProvider.synthesize(text);
        
        if (ttsResult.format === "opus") {
          resource = createAudioResource(Readable.from(ttsResult.audioBuffer), {
            inputType: StreamType.OggOpus,
          });
        } else {
          resource = createAudioResource(Readable.from(ttsResult.audioBuffer));
        }
        
        this.logger.debug?.(`[discord-voice] Using buffered TTS`);
      }

      if (!resource) {
        throw new Error("Failed to create audio resource");
      }

      session.player.play(resource);

      // Wait for playback to finish
      await new Promise<void>((resolve) => {
        const onIdle = () => {
          session.speaking = false;
          session.lastSpokeAt = Date.now(); // Set cooldown timestamp
          session.player.off(AudioPlayerStatus.Idle, onIdle);
          session.player.off("error", onError);
          resolve();
        };
        
        const onError = (error: Error) => {
          this.logger.error(`[discord-voice] Playback error: ${error.message}`);
          session.speaking = false;
          session.lastSpokeAt = Date.now(); // Set cooldown timestamp
          session.player.off(AudioPlayerStatus.Idle, onIdle);
          session.player.off("error", onError);
          resolve();
        };

        session.player.on(AudioPlayerStatus.Idle, onIdle);
        session.player.on("error", onError);
      });
    } catch (error) {
      session.speaking = false;
      session.lastSpokeAt = Date.now(); // Set cooldown timestamp
      throw error;
    }
  }

  /**
   * Start looping thinking sound, returns stop function
   */
  private async startThinkingLoop(session: VoiceSession): Promise<() => void> {
    // Optional UX feature. This lightweight skill does not ship with an audio asset.
    // If you want a thinking sound, add assets/thinking.mp3 and implement a loop here.
    void session;
    return () => {};
  }



  /**
   * Calculate RMS (Root Mean Square) amplitude of audio buffer
   * Used to filter out quiet sounds like keystrokes and background noise
   */
  private calculateRMS(audioBuffer: Buffer): number {
    // Audio is 16-bit signed PCM
    const samples = audioBuffer.length / 2;
    if (samples === 0) return 0;

    let sumSquares = 0;
    for (let i = 0; i < audioBuffer.length; i += 2) {
      const sample = audioBuffer.readInt16LE(i);
      sumSquares += sample * sample;
    }

    return Math.sqrt(sumSquares / samples);
  }


  /**
   * Resolve a Discord "user selector" into a userId.
   * Accepts: userId digits, <@123>, <@!123>, or a name to match against members in the current voice channel.
   */
  private resolveUserId(session: VoiceSession, selector: string): string | null {
    const raw = (selector || "").trim();
    if (!raw) return null;

    // <@123> or <@!123>
    const mention = raw.match(/^<@!?([0-9]+)>$/);
    if (mention) return mention[1];

    // plain digits
    if (/^[0-9]{10,}$/.test(raw)) return raw;

    const channel = session.channelRef;
    if (!channel) return null;

    const needle = raw.toLowerCase();

    // Only consider members currently in the voice channel
    const members = Array.from(channel.members.values());

    // Best-effort match order:
    // 1) exact displayName
    // 2) exact username
    // 3) startsWith displayName/username
    // 4) contains displayName/username
    const exact = members.find(m => (m.displayName || "").toLowerCase() === needle)
      || members.find(m => (m.user?.username || "").toLowerCase() === needle);
    if (exact) return exact.id;

    const starts = members.find(m => (m.displayName || "").toLowerCase().startsWith(needle))
      || members.find(m => (m.user?.username || "").toLowerCase().startsWith(needle));
    if (starts) return starts.id;

    const contains = members.find(m => (m.displayName || "").toLowerCase().includes(needle))
      || members.find(m => (m.user?.username || "").toLowerCase().includes(needle));
    if (contains) return contains.id;

    return null;
  }

  /**
   * If primaryUser is configured, resolve it to an ID when we join the channel.
   */
  private resolvePrimaryUser(session: VoiceSession): void {
    if (!this.config.primaryUser) return;
    const id = this.resolveUserId(session, this.config.primaryUser);
    if (id) {
      session.primaryUserId = id;
      this.logger.info(`[discord-voice] Primary speaker resolved to userId=${id}`);
    } else {
      // If we can't resolve (name not in channel yet), we keep primaryUserId unset.
      // The user can re-join / or say the switch command after they join.
      this.logger.warn(`[discord-voice] Could not resolve primaryUser="${this.config.primaryUser}" in this voice channel yet.`);
    }
  }

  /**
   * Voice-only speaker switching:
   * Primary speaker can say:
   * - "openclaw allow <name>"
   * - "openclaw listen to <name>"
   * - "openclaw let <name> speak"
   * - "openclaw only me" / "openclaw just me" / "openclaw reset"
   */
  private tryHandleSpeakerCommand(session: VoiceSession, userId: string, text: string): { handled: boolean; reply?: string } {
    if (!this.config.allowVoiceSwitch) return { handled: false };
    if (!session.primaryUserId) return { handled: false };
    if (userId !== session.primaryUserId) return { handled: false };

    const wake = (this.config.wakeWord || "openclaw").toLowerCase();
    const t = text.trim().toLowerCase();

    // Must start with wake word (or be very explicit)
    if (!t.startsWith(wake)) return { handled: false };

    // Remove wake word
    const rest = text.trim().slice(wake.length).trim();

    // Reset commands
    if (/^(only\s+me|just\s+me|reset|stop\s+listening|listen\s+to\s+me)\b/i.test(rest)) {
      session.activeSpeakerId = undefined;
      return { handled: true, reply: "Okay. I will only listen to you now." };
    }

    // Allow commands
    const m = rest.match(/^(allow|listen\s+to|let)\s+(.+?)\s*(speak)?\s*$/i);
    if (!m) return { handled: false };

    const targetRaw = (m[2] || "").trim();
    if (!targetRaw) return { handled: true, reply: "Who should I allow?" };

    const targetId = this.resolveUserId(session, targetRaw);
    if (!targetId) {
      return { handled: true, reply: `I couldn't find "${targetRaw}" in this voice channel.` };
    }

    // Allow primary + target
    session.activeSpeakerId = targetId;
    return { handled: true, reply: `Okay. I'll listen to ${targetRaw} now.` };
  }

  /**
   * Check if a user is allowed to use voice
   */
  private isUserAllowed(session: VoiceSession, userId: string): boolean {
    // If primaryUser is set, default to listening ONLY to them.
    // If activeSpeakerId is set, listen to primary + active.
    if (session.primaryUserId) {
      if (userId === session.primaryUserId) return true;
      if (session.activeSpeakerId && userId === session.activeSpeakerId) return true;
      return false;
    }

    // Otherwise fall back to allowedUsers list (user IDs)
    if (this.config.allowedUsers.length === 0) return true;
    return this.config.allowedUsers.includes(userId);
  }

  /**
   * Get session for a guild
   */
  getSession(guildId: string): VoiceSession | undefined {
    return this.sessions.get(guildId);
  }


  /**
   * Manually set the active speaker (in addition to the primary speaker).
   * selector can be a userId, mention (<@id>), or a name of someone currently in the voice channel.
   */
  setActiveSpeaker(guildId: string, selector: string): { primaryUserId?: string; activeSpeakerId?: string } {
    const session = this.sessions.get(guildId);
    if (!session) throw new Error(`No active voice session for guild ${guildId}`);
    if (!session.primaryUserId) {
      // If primary user wasn't resolved yet, try again now.
      this.resolvePrimaryUser(session);
    }
    const id = this.resolveUserId(session, selector);
    if (!id) throw new Error(`Could not resolve "${selector}" to a user in this voice channel`);
    session.activeSpeakerId = id;
    return { primaryUserId: session.primaryUserId, activeSpeakerId: session.activeSpeakerId };
  }

  /**
   * Clear active speaker so the bot listens only to primary speaker (if configured).
   */
  clearActiveSpeaker(guildId: string): { primaryUserId?: string; activeSpeakerId?: string } {
    const session = this.sessions.get(guildId);
    if (!session) throw new Error(`No active voice session for guild ${guildId}`);
    session.activeSpeakerId = undefined;
    return { primaryUserId: session.primaryUserId, activeSpeakerId: session.activeSpeakerId };
  }

  /**
   * Get speaker status for a guild.
   */
  getSpeakerStatus(guildId: string): { primaryUserId?: string; activeSpeakerId?: string } {
    const session = this.sessions.get(guildId);
    if (!session) throw new Error(`No active voice session for guild ${guildId}`);
    return { primaryUserId: session.primaryUserId, activeSpeakerId: session.activeSpeakerId };
  }

  /**
   * Get all active sessions
   */
  getAllSessions(): VoiceSession[] {
    return Array.from(this.sessions.values());
  }

  /**
   * Destroy all connections
   */
  async destroy(): Promise<void> {
    // Close streaming STT
    if (this.streamingSTT) {
      this.streamingSTT.closeAll();
    }

    const guildIds = Array.from(this.sessions.keys());
    for (const guildId of guildIds) {
      await this.leave(guildId);
    }
  }
}


Security scan verdicts — ClawScan: suspicious | static analysis: suspicious | VirusTotal: clean | SkillSpector: suspicious (CRITICAL)