app-asr.js

// This file copies and modifies code
// from https://mdn.github.io/web-dictaphone/scripts/app.js
// and https://gist.github.com/meziantou/edb7217fddfbb70e899e

const urlParams = new URLSearchParams(window.location.search);
const lang = urlParams.get('lang') ?? 'en';
const activeTab = urlParams.get('tab') ?? 'single';

const startBtn = document.getElementById('recordBtn');
const hint = document.getElementById('hint');
const soundClips = document.getElementById('fileInput');
const playAllBtn = document.getElementById('playAllBtn');

let started = false;
let multistreamStarted = false;

let textArea = document.getElementById('results');

let lastResult = '';
let resultList = [];

function clear() {
  resultList = [];
  textArea.value = getDisplayResult();
  textArea.scrollTop = textArea.scrollHeight;  // auto scroll
};

function getDisplayResult() {
  let i = 0;
  let ans = '';
  for (let s in resultList) {
    if (resultList[s] == '') {
      continue;
    }

    ans += '' + i + ': ' + resultList[s] + '\n';
    i += 1;
  }

  if (lastResult.length > 0) {
    ans += '' + i + ': ' + lastResult + '\n';
  }
  return ans;
}

Module = {};
Module.locateFile = function(path, scriptDirectory = '') {
  if (path.endsWith('.js.metadata')) {
    return scriptDirectory + path.replace('.js.metadata', '.json');
  }
  return scriptDirectory + path;
};
Module.setStatus = function(status) {
  const statusElement = document.getElementById('status');
  statusElement.textContent = status;
  if (status === '') {
    statusElement.style.display = 'none';
    document.querySelectorAll('.tab-content').forEach((tabContentElement) => {
      tabContentElement.classList.remove('loading');
    });
  } else {
    statusElement.style.display = 'block';
    document.querySelectorAll('.tab-content').forEach((tabContentElement) => {
      tabContentElement.classList.add('loading');
    });
  }
};
Module.onRuntimeInitialized = function() {
  console.log('inited!');
  //hint.innerText = 'Model loaded! Please click start';

  started = false;

  recognizer = createOnlineRecognizer(Module);
  console.log('recognizer is created!', recognizer);
};

function loadScript(src) {
  const scriptElement = document.createElement('script');
  scriptElement.src = src;
  document.body.append(scriptElement);
}

loadScript('./' + lang + '.js');
loadScript('./sherpa-onnx-wasm-main-asr.js');

let audioCtx;
let mediaStream;

let expectedSampleRate = 16000;
let recordSampleRate;  // the sampleRate of the microphone
let recorder = null;   // the microphone
let leftchannel = [];  // TODO: Use a single channel

let recordingLength = 0;  // number of samples so far

let recognizer = null;
let recognizer_stream = null;

if (navigator.mediaDevices.getUserMedia) {
  console.log('getUserMedia supported.');

  // see https://w3c.github.io/mediacapture-main/#dom-mediadevices-getusermedia
  const constraints = {audio: true};

  let onSuccess = function(stream) {
    if (!audioCtx) {
      audioCtx = new AudioContext({sampleRate: 16000});
    }
    console.log(audioCtx);
    recordSampleRate = audioCtx.sampleRate;
    console.log('sample rate ' + recordSampleRate);

    // creates an audio node from the microphone incoming stream
    mediaStream = audioCtx.createMediaStreamSource(stream);
    console.log('media stream', mediaStream);

    // https://developer.mozilla.org/en-US/docs/Web/API/AudioContext/createScriptProcessor
    // bufferSize: the onaudioprocess event is called when the buffer is full
    var bufferSize = 4096;
    var numberOfInputChannels = 1;
    var numberOfOutputChannels = 2;
    if (audioCtx.createScriptProcessor) {
      recorder = audioCtx.createScriptProcessor(
          bufferSize, numberOfInputChannels, numberOfOutputChannels);
    } else {
      recorder = audioCtx.createJavaScriptNode(
          bufferSize, numberOfInputChannels, numberOfOutputChannels);
    }
    console.log('recorder', recorder);

    recorder.onaudioprocess = function(e) {
      let samples = new Float32Array(e.inputBuffer.getChannelData(0))
      samples = downsampleBuffer(samples, expectedSampleRate);

      if (recognizer_stream == null) {
        recognizer_stream = recognizer.createStream();
      }

      recognizer_stream.acceptWaveform(expectedSampleRate, samples);
      while (recognizer.isReady(recognizer_stream)) {
        recognizer.decode(recognizer_stream);
      }

      let isEndpoint = recognizer.isEndpoint(recognizer_stream);

      let result = recognizer.getResult(recognizer_stream).text;

      if (recognizer.config.modelConfig.paraformer.encoder != '') {
        let tailPaddings = new Float32Array(expectedSampleRate);
        recognizer_stream.acceptWaveform(expectedSampleRate, tailPaddings);
        while (recognizer.isReady(recognizer_stream)) {
          recognizer.decode(recognizer_stream);
        }
        result = recognizer.getResult(recognizer_stream).text;
      }


      if (result.length > 0 && lastResult != result) {
        lastResult = result;
      }

      if (isEndpoint) {
        if (lastResult.length > 0) {
          resultList.push(lastResult);
          lastResult = '';
        }
        recognizer.reset(recognizer_stream);
      }

      textArea.value = getDisplayResult();
      textArea.scrollTop = textArea.scrollHeight;  // auto scroll

      let buf = new Int16Array(samples.length);
      for (var i = 0; i < samples.length; ++i) {
        let s = samples[i];
        if (s >= 1)
          s = 1;
        else if (s <= -1)
          s = -1;

        samples[i] = s;
        buf[i] = s * 32767;
      }

      leftchannel.push(buf);
      recordingLength += bufferSize;
    };

    startBtn.onclick = function() {
      if(started) {
        console.log('recorder stopped');
  
        recorder.disconnect(audioCtx.destination);
        mediaStream.disconnect(recorder);
  
        started = false;
  
        var clipName = new Date().toISOString();
  
        const clipContainer = document.createElement('article');
        const clipLabel = document.createElement('p');
        const audio = document.createElement('audio');
        const deleteButton = document.createElement('button');
        clipContainer.classList.add('clip');
        audio.setAttribute('controls', '');
        deleteButton.textContent = 'Delete';
        deleteButton.className = 'delete';
  
        clipLabel.textContent = clipName;
  
        clipContainer.appendChild(audio);
  
        clipContainer.appendChild(clipLabel);
        clipContainer.appendChild(deleteButton);
        soundClips.appendChild(clipContainer);
  
        audio.controls = true;
        let samples = flatten(leftchannel);
        const blob = toWav(samples);
  
        leftchannel = [];
        const audioURL = window.URL.createObjectURL(blob);
        audio.src = audioURL;
        console.log('recorder stopped');
  
        deleteButton.onclick = function(e) {
          let evtTgt = e.target;
          evtTgt.parentNode.parentNode.removeChild(evtTgt.parentNode);
        };
  
        clipLabel.onclick = function() {
          const existingName = clipLabel.textContent;
          const newClipName = prompt('Enter a new name for your sound clip?');
          if (newClipName === null) {
            clipLabel.textContent = existingName;
          } else {
            clipLabel.textContent = newClipName;
          }
        };
      }
      else {
        mediaStream.connect(recorder);
        recorder.connect(audioCtx.destination);

        console.log('recorder started');

        started = true;
      }
    };
  };

  let onError = function(err) {
    console.log('The following error occured: ' + err);
  };

  navigator.mediaDevices.getUserMedia(constraints).then(onSuccess, onError);
} else {
  console.log('getUserMedia not supported on your browser!');
  alert('getUserMedia not supported on your browser!');
}

playAllBtn.onclick = function() {
    if(!multistreamStarted) {
        multistreamStarted = true;
        playAllBtn.textContent = "Stop All Streams";
        playAllBtn.style.backgroundColor = "#d9534f";

        transcribe(audioSources[lang][0], 'transcript1', 0);
        transcribe(audioSources[lang][1], 'transcript2', 1);
        transcribe(audioSources[lang][2], 'transcript3', 2);
        transcribe(audioSources[lang][3], 'transcript4', 3);
        transcribe(audioSources[lang][4], 'transcript5', 4);
    }
    else {
        audios[0].pause();
        audios[1].pause();
        audios[2].pause();
        audios[3].pause();
        audios[4].pause();
        audios[0].currentTime = 0;
        audios[1].currentTime = 0;
        audios[2].currentTime = 0;
        audios[3].currentTime = 0;
        audios[4].currentTime = 0;

        playAllBtn.textContent = "Play All Streams";
        playAllBtn.style.backgroundColor = "#007bff";

        multistreamStarted = false;
    }
}


// this function is copied/modified from
// https://gist.github.com/meziantou/edb7217fddfbb70e899e
function flatten(listOfSamples) {
  let n = 0;
  for (let i = 0; i < listOfSamples.length; ++i) {
    n += listOfSamples[i].length;
  }
  let ans = new Int16Array(n);

  let offset = 0;
  for (let i = 0; i < listOfSamples.length; ++i) {
    ans.set(listOfSamples[i], offset);
    offset += listOfSamples[i].length;
  }
  return ans;
}

// this function is copied/modified from
// https://gist.github.com/meziantou/edb7217fddfbb70e899e
function toWav(samples) {
  let buf = new ArrayBuffer(44 + samples.length * 2);
  var view = new DataView(buf);

  // http://soundfile.sapp.org/doc/WaveFormat/
  //                   F F I R
  view.setUint32(0, 0x46464952, true);               // chunkID
  view.setUint32(4, 36 + samples.length * 2, true);  // chunkSize
  //                   E V A W
  view.setUint32(8, 0x45564157, true);  // format
                                        //
  //                      t m f
  view.setUint32(12, 0x20746d66, true);          // subchunk1ID
  view.setUint32(16, 16, true);                  // subchunk1Size, 16 for PCM
  view.setUint32(20, 1, true);                   // audioFormat, 1 for PCM
  view.setUint16(22, 1, true);                   // numChannels: 1 channel
  view.setUint32(24, expectedSampleRate, true);  // sampleRate
  view.setUint32(28, expectedSampleRate * 2, true);  // byteRate
  view.setUint16(32, 2, true);                       // blockAlign
  view.setUint16(34, 16, true);                      // bitsPerSample
  view.setUint32(36, 0x61746164, true);              // Subchunk2ID
  view.setUint32(40, samples.length * 2, true);      // subchunk2Size

  let offset = 44;
  for (let i = 0; i < samples.length; ++i) {
    view.setInt16(offset, samples[i], true);
    offset += 2;
  }

  return new Blob([view], {type: 'audio/wav'});
}

// this function is copied from
// https://github.com/awslabs/aws-lex-browser-audio-capture/blob/master/lib/worker.js#L46
function downsampleBuffer(buffer, exportSampleRate) {
  if (exportSampleRate === recordSampleRate) {
    return buffer;
  }
  var sampleRateRatio = recordSampleRate / exportSampleRate;
  var newLength = Math.round(buffer.length / sampleRateRatio);
  var result = new Float32Array(newLength);
  var offsetResult = 0;
  var offsetBuffer = 0;
  while (offsetResult < result.length) {
    var nextOffsetBuffer = Math.round((offsetResult + 1) * sampleRateRatio);
    var accum = 0, count = 0;
    for (var i = offsetBuffer; i < nextOffsetBuffer && i < buffer.length; i++) {
      accum += buffer[i];
      count++;
    }
    result[offsetResult] = accum / count;
    offsetResult++;
    offsetBuffer = nextOffsetBuffer;
  }
  return result;
};

async function processArrayBufferWithASR(arrayBuffer, file) {
  // Check if recognizer is ready.
  if (recognizer === null) {
    console.error("Recognizer not yet initialized! Please wait for WASM to load.");
    //resultsTextarea.value = "Error: Recognizer not ready.";
    return;
  }
  // Create an AudioContext. (On some platforms, creating multiple AudioContexts can be problematic.
  // If needed, consider reusing a global AudioContext.)
  const audioCtx = new (window.AudioContext || window.webkitAudioContext)({
    sampleRate: expectedSampleRate
  });
  try {
    const audioBuffer = await audioCtx.decodeAudioData(arrayBuffer);
    console.log("AudioBuffer decoded. Duration (s):", audioBuffer.duration);
    let channelData = audioBuffer.getChannelData(0);
    console.log("Channel data length:", channelData.length);
    // Downsample if necessary.
    if (audioBuffer.sampleRate !== expectedSampleRate) {
      console.log("Downsampling from", audioBuffer.sampleRate, "to", expectedSampleRate);
      channelData = downsampleBuffer(channelData, expectedSampleRate, audioBuffer.sampleRate);
      console.log("Downsampled channel data length:", channelData.length);
    }
    // Create a new recognizer stream.
    const stream = recognizer.createStream();
    const chunkSize = expectedSampleRate; // assume 1 second worth of samples per chunk.
    for (let i = 0; i < channelData.length; i += chunkSize) {
      const chunk = channelData.subarray(i, i + chunkSize);
      stream.acceptWaveform(expectedSampleRate, chunk);
      while (recognizer.isReady(stream)) {
        recognizer.decode(stream);
      }
    }
    // Flush any tail data if necessary.
    const tail = new Float32Array(expectedSampleRate);
    stream.acceptWaveform(expectedSampleRate, tail);
    while (recognizer.isReady(stream)) {
      recognizer.decode(stream);
    }
    const fileResult = recognizer.getResult(stream).text || "";
    console.log("ASR result for file:", fileResult);
    textArea.value = fileResult;
  } catch (err) {
    console.error("Error decoding audio data:", err);
    //resultsTextarea.value = "Error processing audio: " + err.message;
  }
}

const audios = []
const recorders = []

async function loadAudio(url) {
  try {
    const response = await fetch(url, { mode: "cors" });
    if (!response.ok) throw new Error("Network response was not ok");

    const blob = await response.blob();
    const objectUrl = URL.createObjectURL(blob);
    
    return new Audio(objectUrl);
  } catch (error) {
    console.error("Error loading audio:", error);
  }
}

async function transcribe(url, output, index) {
  let urlLabel = document.createElement('h2');
  //urlLabel.textContent = url;
  let textarea = document.getElementById(output);
  textarea.value = '';
  
  //textarea.readOnly = true;
  //textarea.rows = 10;
  //document.querySelector('#container').append(urlLabel);
  //document.querySelector('#container').append(textarea);

  let lastResult = '';
  let resultList = [];

  function getDisplayResult() {
    let i = 0;
    let ans = '';
    for (let s in resultList) {
      if (resultList[s] == '') {
        continue;
      }

      ans += '' + i + ': ' + resultList[s] + '\n';
      i += 1;
    }

    if (lastResult.length > 0) {
      ans += '' + i + ': ' + lastResult + '\n';
    }
    return ans;
  }

  if(!audios[index]) {
    audios[index] = await loadAudio(url);
  }
  audios[index].play();

  console.log(audioCtx);
  let recordSampleRate = audioCtx.sampleRate;
  console.log('sample rate ' + recordSampleRate);

  // https://developer.mozilla.org/en-US/docs/Web/API/AudioContext/createScriptProcessor
  // bufferSize: the onaudioprocess event is called when the buffer is full
  var bufferSize = 4096;
  var numberOfInputChannels = 1;
  var numberOfOutputChannels = 2;
  if(!recorders[index]) {
    recorders[index] = audioCtx.createScriptProcessor(
        bufferSize, numberOfInputChannels, numberOfOutputChannels);
    let mediaStream = audioCtx.createMediaElementSource(audios[index]);
    mediaStream.connect(recorders[index]);
    recorders[index].connect(audioCtx.destination);
  }

  
  let recognizer_stream = null;

  recorders[index].onaudioprocess = function(e) {
    let samples = new Float32Array(e.inputBuffer.getChannelData(0))
    e.outputBuffer.copyToChannel(samples, 0);

    if (recognizer_stream == null) {
      recognizer_stream = recognizer.createStream();
    }

    recognizer_stream.acceptWaveform(expectedSampleRate, samples);
    while (recognizer.isReady(recognizer_stream)) {
      recognizer.decode(recognizer_stream);
    }

    let isEndpoint = recognizer.isEndpoint(recognizer_stream);

    let result = recognizer.getResult(recognizer_stream).text;

    if (result.length > 0 && lastResult != result) {
      lastResult = result;
    }

    if (isEndpoint) {
      if (lastResult.length > 0) {
        resultList.push(lastResult);
        lastResult = '';
      }
      recognizer.reset(recognizer_stream);
    }

    textarea.value = getDisplayResult();
    textarea.scrollTop = textarea.scrollHeight;  // auto scroll

    let buf = new Int16Array(samples.length);
    for (var i = 0; i < samples.length; ++i) {
      let s = samples[i];
      if (s >= 1)
        s = 1;
      else if (s <= -1)
        s = -1;

      samples[i] = s;
      buf[i] = s * 32767;
    }
  };

  recorders[index]?.addEventListener("recordingStopped", () => {
      console.log("Decoding has stopped.");
      mediaStream.disconnect(recorders[index]);
  });
}

soundClips.addEventListener("change", function (event) {
  if (!event.target.files || !event.target.files[0]) {
    console.log("No file selected.");
    return;
  }
  const file = event.target.files[0];
  console.log("Selected file:", file.name, file.type, file.size, "bytes");
  const reader = new FileReader();
  reader.onload = function (ev) {
    console.log("FileReader onload called.");
    const arrayBuffer = ev.target.result;
    console.log("ArrayBuffer length:", arrayBuffer.byteLength);
    var url = URL.createObjectURL(file);
    transcribe(url, 'results');
    //processArrayBufferWithASR(arrayBuffer, file);
  };
  reader.onerror = function (err) {
    console.error("FileReader error:", err);
  };
  console.log("Starting FileReader.readAsArrayBuffer...");
  reader.readAsArrayBuffer(file);
});

const singleAudioTab = document.getElementById("singleAudioTab");
const multistreamTab = document.getElementById("multistreamTab");
const singleAudioContent = document.getElementById("singleAudioContent");
const multistreamContent = document.getElementById("multistreamContent");

const audioElements = [
  document.getElementById("audio1"),
  document.getElementById("audio2"),
  document.getElementById("audio3"),
  document.getElementById("audio4"),
  document.getElementById("audio5"),
];

const audioSources = {
  "de": [
    "./de1.mp3",
    "./de2.mp3",
    "./de3.mp3",
    "./de4.mp3",
    "./de5.mp3",
  ],
  "en": [
    "./en1.mp3",
    "./en2.mp3",
    "./en3.mp3",
    "./en4.mp3",
    "./en5.mp3",
  ],
  "fr": [
    "./fr1.mp3",
    "./fr2.mp3",
    "./fr3.mp3",
    "./fr4.mp3",
    "./fr5.mp3",
  ],
};

// Tab switching logic
singleAudioTab.addEventListener("click", () => {
  singleAudioTab.classList.add("active");
  multistreamTab.classList.remove("active");
  singleAudioContent.style.display = "block";
  multistreamContent.style.display = "none";

  singleAudioTab.style.borderBottomColor = "#007bff";
  multistreamTab.style.borderBottomColor = "transparent";
  singleAudioTab.style.color = "#007bff";
  multistreamTab.style.color = "#6c757d";

  const url = new URL(window.location.href);
  url.searchParams.set("tab", "single");
  window.history.pushState({}, "", url.toString());
});

multistreamTab.addEventListener("click", () => {
  multistreamTab.classList.add("active");
  singleAudioTab.classList.remove("active");
  multistreamContent.style.display = "block";
  singleAudioContent.style.display = "none";

  multistreamTab.style.borderBottomColor = "#007bff";
  singleAudioTab.style.borderBottomColor = "transparent";
  multistreamTab.style.color = "#007bff";
  singleAudioTab.style.color = "#6c757d";

  const url = new URL(window.location.href);
  url.searchParams.set("tab", "multi");
  window.history.pushState({}, "", url.toString());
});

// Load audio sources
audioElements.forEach((audio, index) => {
  audio.src = audioSources[lang][index];
});

// Microphone recording logic
const recordBtn = document.getElementById("recordBtn");
const outputText = document.getElementById("outputText");
const audioPlayback = document.getElementById("audioPlayback");

let mediaRecorder;
let audioChunks = [];

recordBtn.addEventListener("click", async () => {
  if (!mediaRecorder || mediaRecorder.state === "inactive") {
    try {
      const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
      mediaRecorder = new MediaRecorder(stream);

      mediaRecorder.ondataavailable = (event) => {
        audioChunks.push(event.data);
      };

      mediaRecorder.onstop = () => {
        const audioBlob = new Blob(audioChunks, { type: "audio/wav" });
        audioChunks = [];
        const audioURL = URL.createObjectURL(audioBlob);
        audioPlayback.src = audioURL;
        audioPlayback.style.display = "block";
        //outputText.value = "Recording completed. Playback is ready.";
      };

      mediaRecorder.start();
      recordBtn.textContent = "Stop Recording";
      recordBtn.style.color = "#5cb85c";
      //outputText.value = "Recording...";
    } catch (err) {
      //outputText.value = "Error accessing microphone: " + err.message;
    }
  } else if (mediaRecorder.state === "recording") {
    mediaRecorder.stop();
    recordBtn.textContent = "Use Microphone";
    recordBtn.style.color = "#d9534f";
  }
});

// Function to handle language change and update URL
document.querySelectorAll('input[name="language"]').forEach((input) => {
  input.addEventListener('change', function() {
    const selectedLang = this.value;
    
    // Update URL with new language parameter
    const url = new URL(window.location.href);
    url.searchParams.set('lang', selectedLang);

    // Reload the page with new language setting
    window.location.href = url.toString();
  });
});

document.querySelectorAll('input[name="language"]').forEach((input) => {
  input.checked = input.value === lang;
});

if (activeTab === "multi") {
  multistreamTab.classList.add("active");
  singleAudioTab.classList.remove("active");
  multistreamContent.style.display = "block";
  singleAudioContent.style.display = "none";

  multistreamTab.style.borderBottomColor = "#007bff";
  singleAudioTab.style.borderBottomColor = "transparent";
  multistreamTab.style.color = "#007bff";
  singleAudioTab.style.color = "#6c757d";
}