Anna Sun
squash for release
2485dd8
raw
history blame
7.18 kB
import {TYPING_ANIMATION_DELAY_MS} from './StreamingInterface';
import {getURLParams} from './URLParams';
import audioBuffertoWav from 'audiobuffer-to-wav';
import './StreamingInterface.css';
type StartEndTime = {
start: number;
end: number;
};
type StartEndTimeWithAudio = StartEndTime & {
float32Audio: Float32Array;
};
type Text = {
time: number;
chars: number;
};
type DebugTimings = {
receivedAudio: StartEndTime[];
playedAudio: StartEndTimeWithAudio[];
receivedText: Text[];
renderedText: StartEndTime[];
sentAudio: StartEndTimeWithAudio[];
startRenderTextTime: number | null;
startRecordingTime: number | null;
receivedAudioSampleRate: number | null;
};
function getInitialTimings(): DebugTimings {
return {
receivedAudio: [],
playedAudio: [],
receivedText: [],
renderedText: [],
sentAudio: [],
startRenderTextTime: null,
startRecordingTime: null,
receivedAudioSampleRate: null,
};
}
function downloadAudioBuffer(audioBuffer: AudioBuffer, fileName: string): void {
const wav = audioBuffertoWav(audioBuffer);
const wavBlob = new Blob([new DataView(wav)], {
type: 'audio/wav',
});
const url = URL.createObjectURL(wavBlob);
const anchor = document.createElement('a');
anchor.href = url;
anchor.target = '_blank';
anchor.download = fileName;
anchor.click();
}
// Uncomment for debugging without download
// function playAudioBuffer(audioBuffer: AudioBuffer): void {
// const audioContext = new AudioContext();
// const source = audioContext.createBufferSource();
// source.buffer = audioBuffer;
// source.connect(audioContext.destination);
// source.start();
// }
// Accumulate timings and audio / text translation samples for debugging and exporting
class DebugTimingsManager {
timings: DebugTimings = getInitialTimings();
start(): void {
this.timings = getInitialTimings();
this.timings.startRecordingTime = new Date().getTime();
}
sentAudio(event: AudioProcessingEvent): void {
const end = new Date().getTime();
const start = end - event.inputBuffer.duration * 1000;
// Copy or else buffer seems to be re-used
const float32Audio = new Float32Array(event.inputBuffer.getChannelData(0));
this.timings.sentAudio.push({
start,
end,
float32Audio,
});
}
receivedText(text: string): void {
this.timings.receivedText.push({
time: new Date().getTime(),
chars: text.length,
});
}
startRenderText(): void {
if (this.timings.startRenderTextTime == null) {
this.timings.startRenderTextTime = new Date().getTime();
}
}
endRenderText(): void {
if (this.timings.startRenderTextTime == null) {
console.warn(
'Wrong timings of start / end rendering text. startRenderText is null',
);
return;
}
this.timings.renderedText.push({
start: this.timings.startRenderTextTime as number,
end: new Date().getTime(),
});
this.timings.startRenderTextTime = null;
}
receivedAudio(duration: number): void {
const start = new Date().getTime();
this.timings.receivedAudio.push({
start,
end: start + duration * 1000,
});
}
playedAudio(start: number, end: number, buffer: AudioBuffer | null): void {
if (buffer != null) {
if (this.timings.receivedAudioSampleRate == null) {
this.timings.receivedAudioSampleRate = buffer.sampleRate;
}
if (this.timings.receivedAudioSampleRate != buffer.sampleRate) {
console.error(
'Sample rates of received audio are unequal, will fail to reconstruct debug audio',
this.timings.receivedAudioSampleRate,
buffer.sampleRate,
);
}
}
this.timings.playedAudio.push({
start,
end,
float32Audio:
buffer == null
? new Float32Array()
: new Float32Array(buffer.getChannelData(0)),
});
}
getChartData() {
const columns = [
{type: 'string', id: 'Series'},
{type: 'date', id: 'Start'},
{type: 'date', id: 'End'},
];
return [
columns,
...this.timings.sentAudio.map((sentAudio) => [
'Sent Audio',
new Date(sentAudio.start),
new Date(sentAudio.end),
]),
...this.timings.receivedAudio.map((receivedAudio) => [
'Received Audio',
new Date(receivedAudio.start),
new Date(receivedAudio.end),
]),
...this.timings.playedAudio.map((playedAudio) => [
'Played Audio',
new Date(playedAudio.start),
new Date(playedAudio.end),
]),
// Best estimate duration by multiplying length with animation duration for each letter
...this.timings.receivedText.map((receivedText) => [
'Received Text',
new Date(receivedText.time),
new Date(
receivedText.time + receivedText.chars * TYPING_ANIMATION_DELAY_MS,
),
]),
...this.timings.renderedText.map((renderedText) => [
'Rendered Text',
new Date(renderedText.start),
new Date(renderedText.end),
]),
];
}
downloadInputAudio() {
const audioContext = new AudioContext();
const totalLength = this.timings.sentAudio.reduce((acc, cur) => {
return acc + cur?.float32Audio?.length ?? 0;
}, 0);
if (totalLength === 0) {
return;
}
const incomingArrayBuffer = audioContext.createBuffer(
1, // 1 channel
totalLength,
audioContext.sampleRate,
);
const buffer = incomingArrayBuffer.getChannelData(0);
let i = 0;
this.timings.sentAudio.forEach((sentAudio) => {
sentAudio.float32Audio.forEach((bytes) => {
buffer[i++] = bytes;
});
});
// Play for debugging
// playAudioBuffer(incomingArrayBuffer);
downloadAudioBuffer(incomingArrayBuffer, `input_audio.wav`);
}
downloadOutputAudio() {
const playedAudio = this.timings.playedAudio;
const sampleRate = this.timings.receivedAudioSampleRate;
if (
playedAudio.length === 0 ||
this.timings.startRecordingTime == null ||
sampleRate == null
) {
return null;
}
let previousEndTime = this.timings.startRecordingTime;
const audioArray: number[] = [];
playedAudio.forEach((audio) => {
const delta = (audio.start - previousEndTime) / 1000;
for (let i = 0; i < delta * sampleRate; i++) {
audioArray.push(0.0);
}
audio.float32Audio.forEach((bytes) => audioArray.push(bytes));
previousEndTime = audio.end;
});
const audioContext = new AudioContext();
const incomingArrayBuffer = audioContext.createBuffer(
1, // 1 channel
audioArray.length,
sampleRate,
);
incomingArrayBuffer.copyToChannel(
new Float32Array(audioArray),
0, // first channel
);
// Play for debugging
// playAudioBuffer(incomingArrayBuffer);
downloadAudioBuffer(incomingArrayBuffer, 'output_audio.wav');
}
}
const debugSingleton = new DebugTimingsManager();
export default function debug(): DebugTimingsManager | null {
const debugParam = getURLParams().debug;
return debugParam ? debugSingleton : null;
}