Repository: whitphx/transformers.js.py (language: python)

Issue:
Support 'automatic-speech-recognition' pipeline
The following error occurs when Transformers.js.py is used in a WebWorker (e.g. used with Gradio-lite) because `AudioContext` is not available on the worker context.

```
pyodide.ffi.JsException: Error: Unable to load audio from path/URL since `AudioContext` is not available in your environment. Instead, audio data should be passed directly to the pipeline/processor. For more information and some example code, see https://huggingface.co/docs/transformers.js/tutorials/node-audio-processing.
```

`AudioContext` is used in Transformers.js here:
https://github.com/xenova/transformers.js/blob/e457c4b5d097730a1226da6d62a3f2d7c0a4f192/src/utils/audio.js#L21


Discussion / hints:

Fix patch:
diff --git a/transformers_js_py/__init__.py b/transformers_js_py/__init__.py
index 2e73f30..2312f25 100644
--- a/transformers_js_py/__init__.py
+++ b/transformers_js_py/__init__.py
@@ -1,4 +1,5 @@
+from .audio import read_audio
 from .proxies import import_transformers_js
 from .url import as_url
 
-__all__ = ["as_url", "import_transformers_js"]
+__all__ = ["as_url", "read_audio", "import_transformers_js"]
diff --git a/transformers_js_py/audio.py b/transformers_js_py/audio.py
new file mode 100644
index 0000000..b3dfb6b
--- /dev/null
+++ b/transformers_js_py/audio.py
@@ -0,0 +1,43 @@
+try:
+    import numpy as np
+except ImportError:
+    np = None  # type: ignore
+
+
+def read_audio(filename, sampling_rate: int) -> "np.ndarray":
+    # Refs:
+    # * https://github.com/xenova/transformers.js/blob/2.15.1/src/utils/audio.js#L42-L77
+    # * https://huggingface.co/docs/transformers.js/guides/node-audio-processing
+
+    try:
+        import numpy as np
+        import scipy  # type: ignore
+    except ImportError:
+        raise ImportError(
+            "You need to have `numpy` and `scipy` installed to use this feature."
+        )
+
+    original_sample_rate, samples = scipy.io.wavfile.read(filename, mmap=False)
+
+    # Ensure samples are float32
+    # Ref: https://docs.scipy.org/doc/scipy/reference/generated/scipy.io.wavfile.read.html  # noqa: E501
+    if samples.dtype == np.int16:
+        samples = samples.astype(np.float32) / 32768.0
+    elif samples.dtype == np.int32:
+        samples = samples.astype(np.float32) / 2147483648.0
+    elif samples.dtype == np.uint8:
+        samples = (samples.astype(np.float32) - 128.0) / 128.0
+
+    if original_sample_rate != sampling_rate:
+        samples = scipy.signal.resample(
+            samples, int(len(samples) * sampling_rate / original_sample_rate)
+        )
+
+    if samples.ndim > 1 and samples.shape[1] > 1:
+        SCALING_FACTOR = np.sqrt(2)
+        # Merge channels (into first channel to save memory)
+        left = samples[:, 0]
+        right = samples[:, 1]
+        samples = SCALING_FACTOR * (left + right) / 2
+
+    return samples


Test patch:
diff --git a/pyodide-e2e/src/tests/audio.test.ts b/pyodide-e2e/src/tests/audio.test.ts
new file mode 100644
index 0000000..c8ab5c9
--- /dev/null
+++ b/pyodide-e2e/src/tests/audio.test.ts
@@ -0,0 +1,32 @@
+import type { PyodideInterface } from "pyodide";
+import { beforeEach, describe, it, expect } from "vitest";
+import { setupPyodideForTest } from "./utils";
+
+import { downloadFile } from "./utils";
+
+describe("read_audio() and ASR", () => {
+  let pyodide: PyodideInterface;
+
+  beforeEach(async () => {
+    pyodide = await setupPyodideForTest(["scipy"]);
+  });
+
+  it("can read an audio file from a local file", async () => {
+    await downloadFile(pyodide, "https://huggingface.co/datasets/Xenova/transformers.js-docs/resolve/main/jfk.wav", "/tmp/jfk.wav")
+
+    await pyodide.runPythonAsync(`
+from transformers_js_py import import_transformers_js, read_audio
+import numpy as np
+
+transformers = await import_transformers_js()
+pipeline = transformers.pipeline
+pipe = await pipeline('automatic-speech-recognition', 'Xenova/whisper-tiny.en')
+
+audio = read_audio("/tmp/jfk.wav", 16000)
+result = await pipe(audio)
+text = result["text"]
+`);
+    const text = await pyodide.globals.get("text");
+    expect(text).toEqual(" And so my fellow Americans ask not what your country can do for you, ask what you can do for your country.");
+  });
+});