Repository: modal-labs/modal-examples

Issue:
Example 06_gpu_and_ml/openai_whisper/streaming not working
Hi there,

I directly copy-pasted the example code from [main.py](https://github.com/SeeknnDestroy/modal-examples/blob/main/06_gpu_and_ml/openai_whisper/streaming/main.py) here from modal-examples and deployed it via `modal deploy main.py`

After successful deployment however, i am not able to use fastapi endpoint via curl nor `modal run main.py` because it gives several errors.

One of which:
```
Traceback (most recent call last):
Expand
  File "/pkg/modal/_container_io_manager.py", line 488, in handle_input_exception
    yield
  File "/pkg/modal/_container_entrypoint.py", line 244, in run_input
    async for value in res:
  File "/pkg/modal/_asgi.py", line 126, in fn
    app_task.result()  # consume/raise exceptions if there are any!
  File "/usr/local/lib/python3.10/site-packages/fastapi/applications.py", line 270, in __call__
    await super().__call__(scope, receive, send)
  File "/usr/local/lib/python3.10/site-packages/starlette/applications.py", line 124, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/usr/local/lib/python3.10/site-packages/starlette/middleware/errors.py", line 184, in __call__
    raise exc
  File "/usr/local/lib/python3.10/site-packages/starlette/middleware/errors.py", line 162, in __call__
    await self.app(scope, receive, _send)
  File "/usr/local/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 79, in __call__
    raise exc
  File "/usr/local/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 68, in __call__
    await self.app(scope, receive, sender)
  File "/usr/local/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 21, in __call__
    raise e
  File "/usr/local/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__
    await self.app(scope, receive, send)
  File "/usr/local/lib/python3.10/site-packages/starlette/routing.py", line 706, in __call__
    await route.handle(scope, receive, send)
  File "/usr/local/lib/python3.10/site-packages/starlette/routing.py", line 276, in handle
    await self.app(scope, receive, send)
  File "/usr/local/lib/python3.10/site-packages/starlette/routing.py", line 66, in app
    response = await func(request)
  File "/usr/local/lib/python3.10/site-packages/fastapi/routing.py", line 235, in app
    raw_response = await run_endpoint_function(
  File "/usr/local/lib/python3.10/site-packages/fastapi/routing.py", line 161, in run_endpoint_function
    return await dependant.call(**values)
  File "/root/modal_app.py", line 216, in transcribe
    audio_data = download_mp3_from_youtube(url)
TypeError: 'Function' object is not callable
```

the second error related to `.starmap` usage without async in for loop:
```
downloading https://www.youtube.com/watch?v=s_LncVnecLA
Expand
streaming transcription of https://www.youtube.com/watch?v=s_LncVnecLA audio to client...
Traceback (most recent call last):
  File "/pkg/modal/_container_io_manager.py", line 488, in handle_input_exception
    yield
  File "/pkg/modal/_container_entrypoint.py", line 244, in run_input
    async for value in res:
  File "/pkg/modal/_asgi.py", line 126, in fn
    app_task.result()  # consume/raise exceptions if there are any!
  File "/usr/local/lib/python3.10/site-packages/fastapi/applications.py", line 270, in __call__
    await super().__call__(scope, receive, send)
  File "/usr/local/lib/python3.10/site-packages/starlette/applications.py", line 124, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/usr/local/lib/python3.10/site-packages/starlette/middleware/errors.py", line 184, in __call__
    raise exc
  File "/usr/local/lib/python3.10/site-packages/starlette/middleware/errors.py", line 162, in __call__
    await self.app(scope, receive, _send)
  File "/usr/local/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 79, in __call__
    raise exc
  File "/usr/local/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 68, in __call__
    await self.app(scope, receive, sender)
  File "/usr/local/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 21, in __call__
    raise e
  File "/usr/local/lib/python3.10/site-packages/fastapi/middleware/asyncexitstack.py", line 18, in __call__
    await self.app(scope, receive, send)
  File "/usr/local/lib/python3.10/site-packages/starlette/routing.py", line 706, in __call__
    await route.handle(scope, receive, send)
  File "/usr/local/lib/python3.10/site-packages/starlette/routing.py", line 276, in handle
    await self.app(scope, receive, send)
  File "/usr/local/lib/python3.10/site-packages/starlette/routing.py", line 69, in app
    await response(scope, receive, send)
  File "/usr/local/lib/python3.10/site-packages/starlette/responses.py", line 266, in __call__
    async with anyio.create_task_group() as task_group:
  File "/usr/local/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 597, in __aexit__
    raise exceptions[0]
  File "/usr/local/lib/python3.10/site-packages/starlette/responses.py", line 269, in wrap
    await func()
  File "/usr/local/lib/python3.10/site-packages/starlette/responses.py", line 258, in stream_response
    async for chunk in self.body_iterator:
  File "/root/modal_app.py", line 186, in stream_whisper
    for result in transcribe_segment.starmap(
  File "/pkg/modal/_utils/async_utils.py", line 364, in __iter__
    raise InvalidError(self.nested_async_message)
modal.exception.InvalidError: You can't run Function.map() or Function.for_each() from an async function. Use Function.map.aio()/Function.for_each.aio() instead.
```


Discussion:

Resolving patch:
diff --git a/06_gpu_and_ml/openai_whisper/streaming/main.py b/06_gpu_and_ml/openai_whisper/streaming/main.py
--- a/06_gpu_and_ml/openai_whisper/streaming/main.py
+++ b/06_gpu_and_ml/openai_whisper/streaming/main.py
@@ -183,16 +183,16 @@ async def stream_whisper(audio_data: bytes):
         f.flush()
         segment_gen = split_silences(f.name)
 
-    for result in transcribe_segment.starmap(
+    async for result in transcribe_segment.starmap(
         segment_gen, kwargs=dict(audio_data=audio_data, model="base.en")
     ):
-        # Must cooperatively yeild here otherwise `StreamingResponse` will not iteratively return stream parts.
-        # see: https://github.com/python/asyncio/issues/284
-        await asyncio.sleep(0.5)
+        # Must cooperatively yield here otherwise `StreamingResponse` will not iteratively return stream parts.
+        # see: https://github.com/python/asyncio/issues/284#issuecomment-154162668
+        await asyncio.sleep(0)
         yield result["text"]
 
 
-@web_app.get("/")
+@web_app.get("/transcribe")
 async def transcribe(url: str):
     """
     Usage:
@@ -213,7 +213,7 @@ async def transcribe(url: str):
 
     print(f"downloading {url}")
     try:
-        audio_data = download_mp3_from_youtube(url)
+        audio_data = download_mp3_from_youtube.remote(url)
     except pytube.exceptions.RegexMatchError:
         raise HTTPException(
             status_code=422, detail=f"Could not process url {url}"


Test patch: