how to use llamafactory api with audio?

### Reminder

- [x] I have read the above rules and searched the existing issues.

### System Info

none

### Reproduction

```text
Put your message here.
```


### Others

    headers = {
        'accept': 'application/json',
        'Content-Type': 'application/json'
    }
    data = {
        "model": MODEL_NAME,
        "messages": [
            {
                "role": "system",
                "content": ""
            },
            {
                "role": "user",
                "content": [
                    {
                        "type": "text",
                        "text": "What type of ship does this audio belong to?"
                    },
                    {
                        "type": "audio_url",
                        "audio_url": {
                            "url": audio_file_path
                        }
                    }
                ]
            },

        ],
        "max_tokens": MAX_TOKENS,
        "do_sample": DO_SAMPLE,
        "num_beams": NUM_BEAMS
    } I receive response  like 
ERROR:    Exception in ASGI application
Traceback (most recent call last):
  File "/home/hmc/miniconda3/envs/llamafactory/lib/python3.10/site-packages/uvicorn/protocols/http/h11_impl.py", line 403, in run_asgi
    result = await app(  # type: ignore[func-returns-value]
  File "/home/hmc/miniconda3/envs/llamafactory/lib/python3.10/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in __call__
    return await self.app(scope, receive, send)
  File "/home/hmc/miniconda3/envs/llamafactory/lib/python3.10/site-packages/fastapi/applications.py", line 1054, in __call__
    await super().__call__(scope, receive, send)
  File "/home/hmc/miniconda3/envs/llamafactory/lib/python3.10/site-packages/starlette/applications.py", line 112, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/home/hmc/miniconda3/envs/llamafactory/lib/python3.10/site-packages/starlette/middleware/errors.py", line 187, in __call__
    raise exc
  File "/home/hmc/miniconda3/envs/llamafactory/lib/python3.10/site-packages/starlette/middleware/errors.py", line 165, in __call__
    await self.app(scope, receive, _send)
  File "/home/hmc/miniconda3/envs/llamafactory/lib/python3.10/site-packages/starlette/middleware/cors.py", line 85, in __call__
    await self.app(scope, receive, send)
  File "/home/hmc/miniconda3/envs/llamafactory/lib/python3.10/site-packages/starlette/middleware/exceptions.py", line 62, in __call__
    await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
  File "/home/hmc/miniconda3/envs/llamafactory/lib/python3.10/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
    raise exc
  File "/home/hmc/miniconda3/envs/llamafactory/lib/python3.10/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
    await app(scope, receive, sender)
  File "/home/hmc/miniconda3/envs/llamafactory/lib/python3.10/site-packages/starlette/routing.py", line 714, in __call__
    await self.middleware_stack(scope, receive, send)
  File "/home/hmc/miniconda3/envs/llamafactory/lib/python3.10/site-packages/starlette/routing.py", line 734, in app
    await route.handle(scope, receive, send)
  File "/home/hmc/miniconda3/envs/llamafactory/lib/python3.10/site-packages/starlette/routing.py", line 288, in handle
    await self.app(scope, receive, send)
  File "/home/hmc/miniconda3/envs/llamafactory/lib/python3.10/site-packages/starlette/routing.py", line 76, in app
    await wrap_app_handling_exceptions(app, request)(scope, receive, send)
  File "/home/hmc/miniconda3/envs/llamafactory/lib/python3.10/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
    raise exc
  File "/home/hmc/miniconda3/envs/llamafactory/lib/python3.10/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
    await app(scope, receive, sender)
  File "/home/hmc/miniconda3/envs/llamafactory/lib/python3.10/site-packages/starlette/routing.py", line 73, in app
    response = await f(request)
  File "/home/hmc/miniconda3/envs/llamafactory/lib/python3.10/site-packages/fastapi/routing.py", line 301, in app
    raw_response = await run_endpoint_function(
  File "/home/hmc/miniconda3/envs/llamafactory/lib/python3.10/site-packages/fastapi/routing.py", line 212, in run_endpoint_function
    return await dependant.call(**values)
  File "/home/hmc/llamafactory/LLaMA-Factory-main/src/llamafactory/api/app.py", line 110, in create_chat_completion
    return await create_chat_completion_response(request, chat_model)
  File "/home/hmc/llamafactory/LLaMA-Factory-main/src/llamafactory/api/chat.py", line 189, in create_chat_completion_response
    responses = await chat_model.achat(
  File "/home/hmc/llamafactory/LLaMA-Factory-main/src/llamafactory/chat/chat_model.py", line 92, in achat
    return await self.engine.chat(messages, system, tools, images, videos, audios, **input_kwargs)
  File "/home/hmc/llamafactory/LLaMA-Factory-main/src/llamafactory/chat/hf_engine.py", line 363, in chat
    return await asyncio.to_thread(self._chat, *input_args)
  File "/home/hmc/miniconda3/envs/llamafactory/lib/python3.10/asyncio/threads.py", line 25, in to_thread
    return await loop.run_in_executor(None, func_call)
  File "/home/hmc/miniconda3/envs/llamafactory/lib/python3.10/concurrent/futures/thread.py", line 58, in run
    result = self.fn(*self.args, **self.kwargs)
  File "/home/hmc/miniconda3/envs/llamafactory/lib/python3.10/site-packages/torch/utils/_contextlib.py", line 116, in decorate_context
    return func(*args, **kwargs)
  File "/home/hmc/llamafactory/LLaMA-Factory-main/src/llamafactory/chat/hf_engine.py", line 226, in _chat
    gen_kwargs, prompt_length = HuggingfaceEngine._process_args(
  File "/home/hmc/llamafactory/LLaMA-Factory-main/src/llamafactory/chat/hf_engine.py", line 103, in _process_args
    messages = template.mm_plugin.process_messages(
  File "/home/hmc/llamafactory/LLaMA-Factory-main/src/llamafactory/data/mm_plugin.py", line 1333, in process_messages
    mm_inputs = self._get_mm_inputs([], [], audios, processor)
  File "/home/hmc/llamafactory/LLaMA-Factory-main/src/llamafactory/data/mm_plugin.py", line 381, in _get_mm_inputs
    audios = self._regularize_audios(
  File "/home/hmc/llamafactory/LLaMA-Factory-main/src/llamafactory/data/mm_plugin.py", line 305, in _regularize_audios
    raise ValueError(f"Expect input is a list of audios, but got {type(audio)}.")
ValueError: Expect input is a list of audios, but got <class '_io.BufferedReader'>.


Labels: solved
State: closed