Repository: livekit/agents-js
Pull request #390: Shubhra/ajs 31 refactor vad with streams
Author: login: Shubhrakanti
Created: 2025-05-14T06:35:53Z | Merged: 2025-05-19T21:26:52Z

Refactor the VAD component to use WHATWG streams.

We use two [Identity TransformStreams](https://streams.spec.whatwg.org/#identity-transform-stream) to represent input and output.

Some parts of the `VAD` component are deprecated because of this refactor. Will remove once we do away with `VoicePipelineAgent`. Keeping it around for now because we're still developing. 


Diff:
diff --git a/agents/src/vad.ts b/agents/src/vad.ts
index cc1f0fba7..2e135df3a 100644
--- a/agents/src/vad.ts
+++ b/agents/src/vad.ts
@@ -4,11 +4,15 @@
 import type { AudioFrame } from '@livekit/rtc-node';
 import type { TypedEventEmitter as TypedEmitter } from '@livekit/typed-emitter';
 import { EventEmitter } from 'node:events';
-import type { ReadableStream } from 'node:stream/web';
+import type {
+  ReadableStream,
+  ReadableStreamDefaultReader,
+  WritableStreamDefaultWriter,
+} from 'node:stream/web';
 import { log } from './log.js';
 import type { VADMetrics } from './metrics/base.js';
 import { DeferredReadableStream } from './stream/deferred_stream.js';
-import { AsyncIterableQueue } from './utils.js';
+import { IdentityTransform } from './stream/identity_transform.js';
 
 export enum VADEventType {
   START_OF_SPEECH,
@@ -80,46 +84,70 @@ export abstract class VAD extends (EventEmitter as new () => TypedEmitter<VADCal
 
 export abstract class VADStream implements AsyncIterableIterator<VADEvent> {
   protected static readonly FLUSH_SENTINEL = Symbol('FLUSH_SENTINEL');
-  protected input = new AsyncIterableQueue<AudioFrame | typeof VADStream.FLUSH_SENTINEL>();
-  protected queue = new AsyncIterableQueue<VADEvent>();
-  protected output = new AsyncIterableQueue<VADEvent>();
+  protected input = new IdentityTransform<AudioFrame | typeof VADStream.FLUSH_SENTINEL>();
+  protected output = new IdentityTransform<VADEvent>();
+  protected inputWriter: WritableStreamDefaultWriter<AudioFrame | typeof VADStream.FLUSH_SENTINEL>;
+  protected inputReader: ReadableStreamDefaultReader<AudioFrame | typeof VADStream.FLUSH_SENTINEL>;
+  protected outputWriter: WritableStreamDefaultWriter<VADEvent>;
+  protected outputReader: ReadableStreamDefaultReader<VADEvent>;
   protected closed = false;
+  protected inputClosed = false;
+
   #vad: VAD;
   #lastActivityTime = BigInt(0);
   private logger = log();
   private deferredInputStream: DeferredReadableStream<AudioFrame>;
 
+  private metricsStream: ReadableStream<VADEvent>;
   constructor(vad: VAD) {
     this.#vad = vad;
     this.deferredInputStream = new DeferredReadableStream<AudioFrame>();
+
+    this.inputWriter = this.input.writable.getWriter();
+    this.inputReader = this.input.readable.getReader();
+    this.outputWriter = this.output.writable.getWriter();
+
+    const [outputStream, metricsStream] = this.output.readable.tee();
+    this.metricsStream = metricsStream;
+    this.outputReader = outputStream.getReader();
+
+    this.pumpDeferredStream();
     this.monitorMetrics();
-    this.mainTask();
   }
 
-  protected async mainTask() {
-    // This is just a placeholder since VAD isn't implemented with the streams API yet.
+  /**
+   * Reads from the deferred input stream and forwards chunks to the input writer.
+   *
+   * Note: we can't just do this.deferredInputStream.stream.pipeTo(this.input.writable)
+   * because the inputWriter locks the this.input.writable stream. All writes must go through
+   * the inputWriter.
+   */
+  private async pumpDeferredStream() {
+    const reader = this.deferredInputStream.stream.getReader();
     try {
-      const inputStream = this.deferredInputStream.stream;
-      const reader = inputStream.getReader();
       while (true) {
         const { done, value } = await reader.read();
-        if (done) {
-          break;
-        }
-        this.pushFrame(value);
+        if (done) break;
+        await this.inputWriter.write(value);
       }
-    } catch (error) {
-      this.logger.error('Error in VADStream mainTask:', error);
+    } catch (e) {
+      this.logger.error(`Error pumping deferred stream: ${e}`);
+      throw e;
+    } finally {
+      reader.releaseLock();
     }
   }
 
   protected async monitorMetrics() {
     let inferenceDurationTotal = 0;
     let inferenceCount = 0;
-
-    for await (const event of this.queue) {
-      this.output.put(event);
-      switch (event.type) {
+    const metricsReader = this.metricsStream.getReader();
+    while (true) {
+      const { done, value } = await metricsReader.read();
+      if (done) {
+        break;
+      }
+      switch (value.type) {
         case VADEventType.START_OF_SPEECH:
           inferenceCount++;
           if (inferenceCount >= 1 / this.#vad.capabilities.updateInterval) {
@@ -143,51 +171,56 @@ export abstract class VADStream implements AsyncIterableIterator<VADEvent> {
           break;
       }
     }
-    this.output.close();
   }
 
   updateInputStream(audioStream: ReadableStream<AudioFrame>) {
     this.deferredInputStream.setSource(audioStream);
   }
 
+  /** @deprecated Use `updateInputStream` instead */
   pushFrame(frame: AudioFrame) {
-    if (this.input.closed) {
+    // TODO(AJS-395): remove this method
+    if (this.inputClosed) {
       throw new Error('Input is closed');
     }
     if (this.closed) {
       throw new Error('Stream is closed');
     }
-    this.input.put(frame);
+    this.inputWriter.write(frame);
   }
 
   flush() {
-    if (this.input.closed) {
+    if (this.inputClosed) {
       throw new Error('Input is closed');
     }
     if (this.closed) {
       throw new Error('Stream is closed');
     }
-    this.input.put(VADStream.FLUSH_SENTINEL);
+    this.inputWriter.write(VADStream.FLUSH_SENTINEL);
   }
 
   endInput() {
-    if (this.input.closed) {
+    if (this.inputClosed) {
       throw new Error('Input is closed');
     }
     if (this.closed) {
       throw new Error('Stream is closed');
     }
-    this.input.close();
+    this.inputClosed = true;
+    this.input.writable.close();
   }
 
-  next(): Promise<IteratorResult<VADEvent>> {
-    return this.output.next();
+  async next(): Promise<IteratorResult<VADEvent>> {
+    return this.outputReader.read().then(({ done, value }) => {
+      if (done) {
+        return { done: true, value: undefined };
+      }
+      return { done: false, value };
+    });
   }
 
   close() {
-    this.input.close();
-    this.queue.close();
-    this.output.close();
+    this.input.writable.close();
     this.closed = true;
   }
 
diff --git a/agents/src/voice/audio_recognition.ts b/agents/src/voice/audio_recognition.ts
index 6a6781217..88699e6bf 100644
--- a/agents/src/voice/audio_recognition.ts
+++ b/agents/src/voice/audio_recognition.ts
@@ -48,10 +48,10 @@ export class AudioRecognition {
   async start() {
     const [vadInputStream, sttInputStream] = this.deferredInputStream.stream.tee();
     this.vadStreamProcessor = this.vadTask(vadInputStream).catch((err) => {
-      throw err;
+      this.logger.error(`Error in VAD task: ${err}`);
     });
     this.sttStreamProcessor = this.sttTask(sttInputStream).catch((err) => {
-      throw err;
+      this.logger.error(`Error in STT task: ${err}`);
     });
   }
 
diff --git a/plugins/silero/src/vad.ts b/plugins/silero/src/vad.ts
index 6d38462b5..1e42aef5e 100644
--- a/plugins/silero/src/vad.ts
+++ b/plugins/silero/src/vad.ts
@@ -157,7 +157,12 @@ export class VADStream extends baseStream {
       // used to avoid drift when the sampleRate ratio is not an integer
       let inputCopyRemainingFrac = 0.0;
 
-      for await (const frame of this.input) {
+      while (true) {
+        const { done, value: frame } = await this.inputReader.read();
+        if (done) {
+          break;
+        }
+
         if (typeof frame === 'symbol') {
           continue; // ignore flush sentinel for now
         }
@@ -255,7 +260,7 @@ export class VADStream extends baseStream {
             pubSilenceDuration += inferenceDuration;
           }
 
-          this.queue.put({
+          this.outputWriter.write({
             type: VADEventType.INFERENCE_DONE,
             samplesIndex: pubCurrentSample,
             timestamp: pubTimestamp,
@@ -309,7 +314,7 @@ export class VADStream extends baseStream {
               pubSilenceDuration = 0;
               pubSpeechDuration = speechThresholdDuration;
 
-              this.queue.put({
+              this.outputWriter.write({
                 type: VADEventType.START_OF_SPEECH,
                 samplesIndex: pubCurrentSample,
                 timestamp: pubTimestamp,
@@ -336,7 +341,7 @@ export class VADStream extends baseStream {
               pubSpeechDuration = 0;
               pubSilenceDuration = silenceThresholdDuration;
 
-              this.queue.put({
+              this.outputWriter.write({
                 type: VADEventType.END_OF_SPEECH,
                 samplesIndex: pubCurrentSample,
                 timestamp: pubTimestamp,


Reviews:
user: login: Shubhrakanti
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T05:32:58Z
user: login: Shubhrakanti
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T05:33:47Z
user: login: Shubhrakanti
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T05:34:41Z
user: login: lukasIO
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T06:50:55Z
user: login: lukasIO
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T06:58:11Z
user: login: Shubhrakanti
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T20:15:23Z
user: login: Shubhrakanti
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-15T20:17:05Z
user: login: lukasIO
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-16T11:32:56Z
user: login: Shubhrakanti
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-19T21:12:12Z

Review comments:
diff_hunk: @@ -80,46 +84,73 @@ export abstract class VAD extends (EventEmitter as new () => TypedEmitter<VADCal
 
 export abstract class VADStream implements AsyncIterableIterator<VADEvent> {
path: agents/src/vad.ts
user: login: Shubhrakanti
body: I don't think this should implement `AsyncIterableIterator`. I think it would be better to have a method

```typescript

get stream(): ReadableStream<VADEvent> {
  return this.output.readable
}

```
created_at: 2025-05-15T05:32:58Z
author_association: CONTRIBUTOR
diff_hunk: @@ -47,8 +47,9 @@ export class AgentActivity implements RecognitionHooks {
     this.logger.info('End of speech', ev);
   }
 
+  // eslint-disable-next-line @typescript-eslint/no-unused-vars
   onVADInferenceDone(ev: VADEvent): void {
-    this.logger.info('VAD inference done', ev);
+    // this.logger.info('VAD inference done', ev);
path: agents/src/voice/agent_activity.ts
user: login: Shubhrakanti
body: ignore this. Forgot to pick this up form another PR's changes. Will resolve with merge conflict
created_at: 2025-05-15T05:33:47Z
author_association: CONTRIBUTOR
diff_hunk: @@ -37,7 +37,9 @@ export class AudioRecognition {
 
   start() {
     this.vadStreamProcessor = this.vadTask().catch((err) => {
-      this.logger.error('Error in VAD task', err);
+      // raise the error
+      throw err;
+      this.logger.error(`Error in VAD task: ${err}`);
     });
path: agents/src/voice/audio_recognition.ts
user: login: Shubhrakanti
body: What's the proper way to handle errors from these async functions we run in node? I'll do some digging on this. Ideally we want to raise them rather than swallow them.
created_at: 2025-05-15T05:34:41Z
author_association: CONTRIBUTOR
diff_hunk: @@ -80,46 +84,73 @@ export abstract class VAD extends (EventEmitter as new () => TypedEmitter<VADCal
 
 export abstract class VADStream implements AsyncIterableIterator<VADEvent> {
path: agents/src/vad.ts
user: login: lukasIO
body: can you elaborate on why (out of curiousity)?
created_at: 2025-05-15T06:50:55Z
author_association: CONTRIBUTOR
diff_hunk: @@ -37,7 +37,9 @@ export class AudioRecognition {
 
   start() {
     this.vadStreamProcessor = this.vadTask().catch((err) => {
-      this.logger.error('Error in VAD task', err);
+      // raise the error
+      throw err;
path: agents/src/voice/audio_recognition.ts
user: login: lukasIO
body: this will always throw as an unhandled exception, which we'd want to avoid. 

You're trying to handle an error in an async task from within a non-async method. I think `catch` is the best you can do there
created_at: 2025-05-15T06:58:11Z
author_association: CONTRIBUTOR
diff_hunk: @@ -80,46 +84,73 @@ export abstract class VAD extends (EventEmitter as new () => TypedEmitter<VADCal
 
 export abstract class VADStream implements AsyncIterableIterator<VADEvent> {
path: agents/src/vad.ts
user: login: Shubhrakanti
body: Yeah this isn't a huge issue for the VAD, but the other components (LLM/TTS/STT) expect the output to be a `ReadableStream` for their default implementations (see [this comment](https://github.com/livekit/agents-js/pull/385/files#r2087921711)). This way the default usage of the components can just be 

```typescript
static default = {
     async [stt/llm/tts]Node(inputStream: ReadableStream<...>): Promise<ReadableStream<...>> {
          ...
          stream = [stt/llm/tts].stream()
          stream.updateInputStream(inputStream)
          return stream.stream()
     }
}
```

[Reference python implementation ](https://github.com/livekit/agents/blob/e57f6a0a4cf07190ca4b60aff6acdadc2cebce90/livekit-agents/livekit/agents/voice/agent.py#L411-L443)

I just want to keep all the `[STT/LLM/TTS/VAD]Stream` classes similar.

created_at: 2025-05-15T20:15:23Z
author_association: CONTRIBUTOR
diff_hunk: @@ -80,46 +84,73 @@ export abstract class VAD extends (EventEmitter as new () => TypedEmitter<VADCal
 
 export abstract class VADStream implements AsyncIterableIterator<VADEvent> {
path: agents/src/vad.ts
user: login: Shubhrakanti
body: Although in this case it makes more sense for `VADStream` to be called `VADStreamManager` or something. 
created_at: 2025-05-15T20:17:05Z
author_association: CONTRIBUTOR
diff_hunk: @@ -80,46 +84,73 @@ export abstract class VAD extends (EventEmitter as new () => TypedEmitter<VADCal
 
 export abstract class VADStream implements AsyncIterableIterator<VADEvent> {
path: agents/src/vad.ts
user: login: lukasIO
body: I see, that makes sense to me!
created_at: 2025-05-16T11:32:56Z
author_association: CONTRIBUTOR
diff_hunk: @@ -80,46 +84,73 @@ export abstract class VAD extends (EventEmitter as new () => TypedEmitter<VADCal
 
 export abstract class VADStream implements AsyncIterableIterator<VADEvent> {
path: agents/src/vad.ts
user: login: Shubhrakanti
body: I'll do this in a separate PR. There are some tricky things with the legacy implementation not worth wrestling right now since we will delete that anyway. https://linear.app/livekit/issue/AJS-48/remove-iterator-from-vadsream
created_at: 2025-05-19T21:12:07Z
author_association: CONTRIBUTOR

PR comments:
user: login: changeset-bot[bot]
created_at: 2025-05-14T06:35:56Z
body: ###  ⚠️  No Changeset found

Latest commit: c4a4fa1da444827ac37539c5429af3149466c92e

Merging this PR will not cause a version bump for any packages. If these changes should not result in a new version, you're good to go. **If these changes should result in a version bump, you need to add a changeset.**

<details><summary>This PR includes no changesets</summary>

  When changesets are added to this PR, you'll see the packages that this PR includes changesets for and the associated semver types

</details>

[Click here to learn what changesets are, and how to add one](https://github.com/changesets/changesets/blob/main/docs/adding-a-changeset.md).

[Click here if you're a maintainer who wants to add a changeset to this PR](https://github.com/livekit/agents-js/new/shubhra/ajs-31-refactor-vad-with-streams?filename=.changeset/stale-hotels-tickle.md&value=---%0A%22%40livekit%2Fagents%22%3A%20patch%0A%22%40livekit%2Fagents-plugin-silero%22%3A%20patch%0A---%0A%0AShubhra%2Fajs%2031%20refactor%20vad%20with%20streams%0A)


author_association: NONE
user: login: Shubhrakanti
created_at: 2025-05-15T05:28:25Z
body: I tried to get this to work by implementing a `VadSrouce` (similar to how we did Audio/VideoStream in `node-sdk`) but was struggling to get it working.

You can see my progress on this commit. https://github.com/livekit/agents-js/pull/390/commits/dd1436262c3df8847130de3fe6c11412ac5bd08f
author_association: CONTRIBUTOR
user: login: Shubhrakanti
created_at: 2025-05-16T15:51:30Z
body: Refactor the streaming logic into a clean interface. The streaming logic is similar to what we need for LLM And TTS and STT. https://linear.app/livekit/issue/AJS-43/stream-wrapper-that-takes-multiple-sources-and-combine-them
author_association: CONTRIBUTOR