Repository: BerriAI/litellm

Issue:
[Bug]: `APIError` crash with DeepSeek R1 getting `LegacyAPIResponse`
### What happened?

Running the below code with Python 3.12.8:

```python
litellm.completion("deepseek/deepseek-reasoner", messages=[{"role": "user", "content": "What is 2+2?"}], max_tokens=4096)
```

LiteLLM crashes with:

```none
APIError: litellm.APIError: APIError: DeepseekException - error - Expecting value: line 1 column 1 (char 0), Received response - <APIResponse [200 OK] type=<class 'openai.types.chat.chat_completion.ChatCompletion'>>, Type of response - <class 'openai._legacy_response.LegacyAPIResponse'>
```

### Relevant log output

```shell
Give Feedback / Get Help: https://github.com/BerriAI/litellm/issues/new
LiteLLM.Info: If you need to debug this error, use `litellm._turn_on_debug()'.


Provider List: https://docs.litellm.ai/docs/providers

---------------------------------------------------------------------------
JSONDecodeError                           Traceback (most recent call last)
File ~/path/to/repo/.venv/lib/python3.12/site-packages/litellm/llms/openai/openai.py:443, in OpenAIChatCompletion.make_sync_openai_chat_completion_request(self, openai_client, data, timeout, logging_obj)
    442     headers = {}
--> 443 response = raw_response.parse()
    444 return headers, response

File ~/path/to/repo/.venv/lib/python3.12/site-packages/openai/_legacy_response.py:137, in LegacyAPIResponse.parse(self, to)
    135     return cached  # type: ignore[no-any-return]
--> 137 parsed = self._parse(to=to)
    138 if is_given(self._options.post_parser):

File ~/path/to/repo/.venv/lib/python3.12/site-packages/openai/_legacy_response.py:332, in LegacyAPIResponse._parse(self, to)
    330     return response.text  # type: ignore
--> 332 data = response.json()
    334 return self._client._process_response_data(
    335     data=data,
    336     cast_to=cast_to,  # type: ignore
    337     response=response,
    338 )

File ~/path/to/repo/.venv/lib/python3.12/site-packages/httpx/_models.py:766, in Response.json(self, **kwargs)
    765 def json(self, **kwargs: typing.Any) -> typing.Any:
--> 766     return jsonlib.loads(self.content, **kwargs)

File ~/.pyenv/versions/3.12.8/lib/python3.12/json/__init__.py:346, in loads(s, cls, object_hook, parse_float, parse_int, parse_constant, object_pairs_hook, **kw)
    343 if (cls is None and object_hook is None and
    344         parse_int is None and parse_float is None and
    345         parse_constant is None and object_pairs_hook is None and not kw):
--> 346     return _default_decoder.decode(s)
    347 if cls is None:

File ~/.pyenv/versions/3.12.8/lib/python3.12/json/decoder.py:338, in JSONDecoder.decode(self, s, _w)
    334 """Return the Python representation of ``s`` (a ``str`` instance
    335 containing a JSON document).
    336
    337 """
--> 338 obj, end = self.raw_decode(s, idx=_w(s, 0).end())
    339 end = _w(s, end).end()

File ~/.pyenv/versions/3.12.8/lib/python3.12/json/decoder.py:356, in JSONDecoder.raw_decode(self, s, idx)
    355 except StopIteration as err:
--> 356     raise JSONDecodeError("Expecting value", s, err.value) from None
    357 return obj, end

JSONDecodeError: Expecting value: line 1 column 1 (char 0)

During handling of the above exception, another exception occurred:

Exception                                 Traceback (most recent call last)
File ~/path/to/repo/.venv/lib/python3.12/site-packages/litellm/llms/openai/openai.py:707, in OpenAIChatCompletion.completion(self, model_response, timeout, optional_params, litellm_params, logging_obj, model, messages, print_verbose, api_key, api_base, api_version, dynamic_params, azure_ad_token, acompletion, logger_fn, headers, custom_prompt_dict, client, organization, custom_llm_provider, drop_params)
    706             else:
--> 707                 raise e
    708 except OpenAIError as e:

File ~/path/to/repo/.venv/lib/python3.12/site-packages/litellm/llms/openai/openai.py:634, in OpenAIChatCompletion.completion(self, model_response, timeout, optional_params, litellm_params, logging_obj, model, messages, print_verbose, api_key, api_base, api_version, dynamic_params, azure_ad_token, acompletion, logger_fn, headers, custom_prompt_dict, client, organization, custom_llm_provider, drop_params)
    622 logging_obj.pre_call(
    623     input=messages,
    624     api_key=openai_client.api_key,
   (...)
    630     },
    631 )
    633 headers, response = (
--> 634     self.make_sync_openai_chat_completion_request(
    635         openai_client=openai_client,
    636         data=data,
    637         timeout=timeout,
    638         logging_obj=logging_obj,
    639     )
    640 )
    642 logging_obj.model_call_details["response_headers"] = headers

File ~/path/to/repo/.venv/lib/python3.12/site-packages/litellm/litellm_core_utils/logging_utils.py:145, in track_llm_api_timing.<locals>.decorator.<locals>.sync_wrapper(*args, **kwargs)
    144 try:
--> 145     result = func(*args, **kwargs)
    146     return result

File ~/path/to/repo/.venv/lib/python3.12/site-packages/litellm/llms/openai/openai.py:447, in OpenAIChatCompletion.make_sync_openai_chat_completion_request(self, openai_client, data, timeout, logging_obj)
    446 if raw_response is not None:
--> 447     raise Exception(
    448         "error - {}, Received response - {}, Type of response - {}".format(
    449             e, raw_response, type(raw_response)
    450         )
    451     )
    452 else:

Exception: error - Expecting value: line 1 column 1 (char 0), Received response - <APIResponse [200 OK] type=<class 'openai.types.chat.chat_completion.ChatCompletion'>>, Type of response - <class 'openai._legacy_response.LegacyAPIResponse'>

During handling of the above exception, another exception occurred:

OpenAIError                               Traceback (most recent call last)
File ~/path/to/repo/.venv/lib/python3.12/site-packages/litellm/main.py:1663, in completion(model, messages, timeout, temperature, top_p, n, stream, stream_options, stop, max_completion_tokens, max_tokens, modalities, prediction, audio, presence_penalty, frequency_penalty, logit_bias, user, response_format, seed, tools, tool_choice, logprobs, top_logprobs, parallel_tool_calls, deployment_id, extra_headers, functions, function_call, base_url, api_version, api_key, model_list, **kwargs)
   1657     logging.post_call(
   1658         input=messages,
   1659         api_key=api_key,
   1660         original_response=str(e),
   1661         additional_args={"headers": headers},
   1662     )
-> 1663     raise e
   1665 if optional_params.get("stream", False):
   1666     ## LOGGING

File ~/path/to/repo/.venv/lib/python3.12/site-packages/litellm/main.py:1636, in completion(model, messages, timeout, temperature, top_p, n, stream, stream_options, stop, max_completion_tokens, max_tokens, modalities, prediction, audio, presence_penalty, frequency_penalty, logit_bias, user, response_format, seed, tools, tool_choice, logprobs, top_logprobs, parallel_tool_calls, deployment_id, extra_headers, functions, function_call, base_url, api_version, api_key, model_list, **kwargs)
   1635 try:
-> 1636     response = openai_chat_completions.completion(
   1637         model=model,
   1638         messages=messages,
   1639         headers=headers,
   1640         model_response=model_response,
   1641         print_verbose=print_verbose,
   1642         api_key=api_key,
   1643         api_base=api_base,
   1644         acompletion=acompletion,
   1645         logging_obj=logging,
   1646         optional_params=optional_params,
   1647         litellm_params=litellm_params,
   1648         logger_fn=logger_fn,
   1649         timeout=timeout,  # type: ignore
   1650         custom_prompt_dict=custom_prompt_dict,
   1651         client=client,  # pass AsyncOpenAI, OpenAI client
   1652         organization=organization,
   1653         custom_llm_provider=custom_llm_provider,
   1654     )
   1655 except Exception as e:
   1656     ## LOGGING - log the original exception returned

File ~/path/to/repo/.venv/lib/python3.12/site-packages/litellm/llms/openai/openai.py:717, in OpenAIChatCompletion.completion(self, model_response, timeout, optional_params, litellm_params, logging_obj, model, messages, print_verbose, api_key, api_base, api_version, dynamic_params, azure_ad_token, acompletion, logger_fn, headers, custom_prompt_dict, client, organization, custom_llm_provider, drop_params)
    716     error_headers = getattr(error_response, "headers", None)
--> 717 raise OpenAIError(
    718     status_code=status_code, message=error_text, headers=error_headers
    719 )

OpenAIError: error - Expecting value: line 1 column 1 (char 0), Received response - <APIResponse [200 OK] type=<class 'openai.types.chat.chat_completion.ChatCompletion'>>, Type of response - <class 'openai._legacy_response.LegacyAPIResponse'>

During handling of the above exception, another exception occurred:

APIError                                  Traceback (most recent call last)
Cell In[21], line 1
----> 1 response = litellm.completion("deepseek/deepseek-reasoner", messages=[{"role": "user", "content": "What is 2+2?"}], max_tokens=4096)

File ~/path/to/repo/.venv/lib/python3.12/site-packages/litellm/utils.py:1143, in client.<locals>.wrapper(*args, **kwargs)
   1139 if logging_obj:
   1140     logging_obj.failure_handler(
   1141         e, traceback_exception, start_time, end_time
   1142     )  # DO NOT MAKE THREADED - router retry fallback relies on this!
-> 1143 raise e

File ~/path/to/repo/.venv/lib/python3.12/site-packages/litellm/utils.py:1021, in client.<locals>.wrapper(*args, **kwargs)
   1019         print_verbose(f"Error while checking max token limit: {str(e)}")
   1020 # MODEL CALL
-> 1021 result = original_function(*args, **kwargs)
   1022 end_time = datetime.datetime.now()
   1023 if "stream" in kwargs and kwargs["stream"] is True:

File ~/path/to/repo/.venv/lib/python3.12/site-packages/litellm/main.py:3018, in completion(model, messages, timeout, temperature, top_p, n, stream, stream_options, stop, max_completion_tokens, max_tokens, modalities, prediction, audio, presence_penalty, frequency_penalty, logit_bias, user, response_format, seed, tools, tool_choice, logprobs, top_logprobs, parallel_tool_calls, deployment_id, extra_headers, functions, function_call, base_url, api_version, api_key, model_list, **kwargs)
   3015     return response
   3016 except Exception as e:
   3017     ## Map to OpenAI Exception
-> 3018     raise exception_type(
   3019         model=model,
   3020         custom_llm_provider=custom_llm_provider,
   3021         original_exception=e,
   3022         completion_kwargs=args,
   3023         extra_kwargs=kwargs,
   3024     )

File ~/path/to/repo/.venv/lib/python3.12/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py:2190, in exception_type(model, original_exception, custom_llm_provider, completion_kwargs, extra_kwargs)
   2188 if exception_mapping_worked:
   2189     setattr(e, "litellm_response_headers", litellm_response_headers)
-> 2190     raise e
   2191 else:
   2192     for error_type in litellm.LITELLM_EXCEPTION_TYPES:

File ~/path/to/repo/.venv/lib/python3.12/site-packages/litellm/litellm_core_utils/exception_mapping_utils.py:450, in exception_type(model, original_exception, custom_llm_provider, completion_kwargs, extra_kwargs)
    448     else:
    449         exception_mapping_worked = True
--> 450         raise APIError(
    451             status_code=original_exception.status_code,
    452             message=f"APIError: {exception_provider} - {message}",
    453             llm_provider=custom_llm_provider,
    454             model=model,
    455             request=getattr(original_exception, "request", None),
    456             litellm_debug_info=extra_information,
    457         )
    458 else:
    459     # if no status code then it is an APIConnectionError: https://github.com/openai/openai-python#handling-errors
    460     # exception_mapping_worked = True
    461     raise APIConnectionError(
    462         message=f"APIConnectionError: {exception_provider} - {message}",
    463         llm_provider=custom_llm_provider,
   (...)
    468         ),
    469     )

APIError: litellm.APIError: APIError: DeepseekException - error - Expecting value: line 1 column 1 (char 0), Received response - <APIResponse [200 OK] type=<class 'openai.types.chat.chat_completion.ChatCompletion'>>, Type of response - <class 'openai._legacy_response.LegacyAPIResponse'>
```

### Are you a ML Ops Team?

No

### What LiteLLM version are you on ?

v1.59.9

### Twitter / LinkedIn details

_No response_

Discussion:
@ishaan-jaff: we'll need to move this to our httpx handler - picking this up tomorrow @jamesbraza  @krrishdholakia: @ishaan-jaff still planning on doing this? @ishaan-jaff: feel free to pick up if you have bandwidth @krrishdholakia  @glenn-jocher: Same exact error here with R1 on latest litellm.@krrishdholakia: able to repro @datalee: Same exact error here with local vllm servers on latest litellm.


Solution patch:
diff --git a/litellm/__init__.py b/litellm/__init__.py
index 0bcd56386..42ed48a47 100644
--- a/litellm/__init__.py
+++ b/litellm/__init__.py
@@ -2,7 +2,7 @@
 import warnings
 
 warnings.filterwarnings("ignore", message=".*conflict with protected namespace.*")
-### INIT VARIABLES #######
+### INIT VARIABLES ########
 import threading
 import os
 from typing import Callable, List, Optional, Dict, Union, Any, Literal, get_args
diff --git a/litellm/litellm_core_utils/get_llm_provider_logic.py b/litellm/litellm_core_utils/get_llm_provider_logic.py
index 302865629..a64e7dd70 100644
--- a/litellm/litellm_core_utils/get_llm_provider_logic.py
+++ b/litellm/litellm_core_utils/get_llm_provider_logic.py
@@ -490,6 +490,7 @@ def _get_openai_compatible_provider_info(  # noqa: PLR0915
             or get_secret("DEEPSEEK_API_BASE")
             or "https://api.deepseek.com/beta"
         )  # type: ignore
+
         dynamic_api_key = api_key or get_secret_str("DEEPSEEK_API_KEY")
     elif custom_llm_provider == "fireworks_ai":
         # fireworks is openai compatible, we just need to set this to custom_openai and have the api_base be https://api.fireworks.ai/inference/v1
diff --git a/litellm/llms/openai/chat/gpt_transformation.py b/litellm/llms/openai/chat/gpt_transformation.py
index 6fa43cccb..98c3254da 100644
--- a/litellm/llms/openai/chat/gpt_transformation.py
+++ b/litellm/llms/openai/chat/gpt_transformation.py
@@ -2,16 +2,27 @@
 Support for gpt model family 
 """
 
-from typing import TYPE_CHECKING, Any, List, Optional, Union, cast
+from typing import (
+    TYPE_CHECKING,
+    Any,
+    AsyncIterator,
+    Iterator,
+    List,
+    Optional,
+    Union,
+    cast,
+)
 
 import httpx
 
 import litellm
+from litellm.llms.base_llm.base_model_iterator import BaseModelResponseIterator
 from litellm.llms.base_llm.base_utils import BaseLLMModelInfo
 from litellm.llms.base_llm.chat.transformation import BaseConfig, BaseLLMException
 from litellm.secret_managers.main import get_secret_str
 from litellm.types.llms.openai import AllMessageValues
-from litellm.types.utils import ModelResponse
+from litellm.types.utils import ModelResponse, ModelResponseStream
+from litellm.utils import convert_to_model_response_object
 
 from ..common_utils import OpenAIError
 
@@ -210,7 +221,36 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
         Returns:
             dict: The transformed response.
         """
-        raise NotImplementedError
+
+        ## LOGGING
+        logging_obj.post_call(
+            input=messages,
+            api_key=api_key,
+            original_response=raw_response.text,
+            additional_args={"complete_input_dict": request_data},
+        )
+
+        ## RESPONSE OBJECT
+        try:
+            completion_response = raw_response.json()
+        except Exception as e:
+            response_headers = getattr(raw_response, "headers", None)
+            raise OpenAIError(
+                message="Unable to get json response - {}, Original Response: {}".format(
+                    str(e), raw_response.text
+                ),
+                status_code=raw_response.status_code,
+                headers=response_headers,
+            )
+        raw_response_headers = dict(raw_response.headers)
+        final_response_obj = convert_to_model_response_object(
+            response_object=completion_response,
+            model_response_object=model_response,
+            hidden_params={"headers": raw_response_headers},
+            _response_headers=raw_response_headers,
+        )
+
+        return cast(ModelResponse, final_response_obj)
 
     def get_error_class(
         self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]
@@ -221,6 +261,30 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
             headers=cast(httpx.Headers, headers),
         )
 
+    def get_complete_url(
+        self,
+        api_base: str,
+        model: str,
+        optional_params: dict,
+        stream: Optional[bool] = None,
+    ) -> str:
+        """
+        Get the complete URL for the API call.
+
+        Returns:
+            str: The complete URL for the API call.
+        """
+        endpoint = "chat/completions"
+
+        # Remove trailing slash from api_base if present
+        api_base = api_base.rstrip("/")
+
+        # Check if endpoint is already in the api_base
+        if endpoint in api_base:
+            return api_base
+
+        return f"{api_base}/{endpoint}"
+
     def validate_environment(
         self,
         headers: dict,
@@ -230,7 +294,14 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
         api_key: Optional[str] = None,
         api_base: Optional[str] = None,
     ) -> dict:
-        raise NotImplementedError
+        if api_key is not None:
+            headers["Authorization"] = f"Bearer {api_key}"
+
+        # Ensure Content-Type is set to application/json
+        if "content-type" not in headers and "Content-Type" not in headers:
+            headers["Content-Type"] = "application/json"
+
+        return headers
 
     def get_models(
         self, api_key: Optional[str] = None, api_base: Optional[str] = None
@@ -272,3 +343,30 @@ class OpenAIGPTConfig(BaseLLMModelInfo, BaseConfig):
             or get_secret_str("OPENAI_API_BASE")
             or "https://api.openai.com/v1"
         )
+
+    def get_model_response_iterator(
+        self,
+        streaming_response: Union[Iterator[str], AsyncIterator[str], ModelResponse],
+        sync_stream: bool,
+        json_mode: Optional[bool] = False,
+    ) -> Any:
+        return OpenAIChatCompletionStreamingHandler(
+            streaming_response=streaming_response,
+            sync_stream=sync_stream,
+            json_mode=json_mode,
+        )
+
+
+class OpenAIChatCompletionStreamingHandler(BaseModelResponseIterator):
+
+    def chunk_parser(self, chunk: dict) -> ModelResponseStream:
+        try:
+            return ModelResponseStream(
+                id=chunk["id"],
+                object="chat.completion.chunk",
+                created=chunk["created"],
+                model=chunk["model"],
+                choices=chunk["choices"],
+            )
+        except Exception as e:
+            raise e
diff --git a/litellm/main.py b/litellm/main.py
index cc71d3133..a6171ec9e 100644
--- a/litellm/main.py
+++ b/litellm/main.py
@@ -1370,6 +1370,36 @@ def completion(  # type: ignore # noqa: PLR0915
                         "api_base": api_base,
                     },
                 )
+        elif custom_llm_provider == "deepseek":
+            ## COMPLETION CALL
+            try:
+                response = base_llm_http_handler.completion(
+                    model=model,
+                    messages=messages,
+                    headers=headers,
+                    model_response=model_response,
+                    api_key=api_key,
+                    api_base=api_base,
+                    acompletion=acompletion,
+                    logging_obj=logging,
+                    optional_params=optional_params,
+                    litellm_params=litellm_params,
+                    timeout=timeout,  # type: ignore
+                    client=client,
+                    custom_llm_provider=custom_llm_provider,
+                    encoding=encoding,
+                    stream=stream,
+                )
+            except Exception as e:
+                ## LOGGING - log the original exception returned
+                logging.post_call(
+                    input=messages,
+                    api_key=api_key,
+                    original_response=str(e),
+                    additional_args={"headers": headers},
+                )
+                raise e
+
         elif custom_llm_provider == "azure_ai":
             api_base = (
                 api_base  # for deepinfra/perplexity/anyscale/groq/friendliai we check in get_llm_provider and pass in the api base from there
@@ -1611,7 +1641,6 @@ def completion(  # type: ignore # noqa: PLR0915
             or custom_llm_provider == "cerebras"
             or custom_llm_provider == "sambanova"
             or custom_llm_provider == "volcengine"
-            or custom_llm_provider == "deepseek"
             or custom_llm_provider == "anyscale"
             or custom_llm_provider == "mistral"
             or custom_llm_provider == "openai"
diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json
index 987ef948a..03eca86e9 100644
--- a/litellm/model_prices_and_context_window_backup.json
+++ b/litellm/model_prices_and_context_window_backup.json
@@ -8904,5 +8904,16 @@
         "supports_function_calling": true,
         "mode": "chat",
         "supports_tool_choice": true
+    },
+    "hyperbolic/deepseek-v3": {
+        "max_tokens": 20480,
+        "max_input_tokens": 131072,
+        "max_output_tokens": 20480,
+        "litellm_provider": "openai",
+        "input_cost_per_token": 0.00000025,
+        "output_cost_per_token": 0.00000025,
+        "mode": "chat",
+        "supports_function_calling": true,
+        "supports_response_schema": true
     }
 }


Test patch:
diff --git a/tests/llm_translation/test_deepseek_completion.py b/tests/llm_translation/test_deepseek_completion.py
index 000c02d67..a07bf3ffe 100644
--- a/tests/llm_translation/test_deepseek_completion.py
+++ b/tests/llm_translation/test_deepseek_completion.py
@@ -7,7 +7,7 @@ import pytest
 class TestDeepSeekChatCompletion(BaseLLMChatTest):
     def get_base_completion_call_args(self) -> dict:
         return {
-            "model": "deepseek/deepseek-chat",
+            "model": "deepseek/deepseek-reasoner",
         }
 
     def test_tool_call_no_arguments(self, tool_call_no_arguments):
@@ -21,3 +21,52 @@ class TestDeepSeekChatCompletion(BaseLLMChatTest):
         Todo: if litellm.modify_params is True ensure it's a valid utf-8 sequence
         """
         pass
+
+
+@pytest.mark.parametrize("stream", [True, False])
+def test_deepseek_mock_completion(stream):
+    """
+    Deepseek API is hanging. Mock the call, to a fake endpoint, so we can confirm our integration is working.
+    """
+    import litellm
+    from litellm import completion
+
+    litellm._turn_on_debug()
+
+    response = completion(
+        model="deepseek/deepseek-reasoner",
+        messages=[{"role": "user", "content": "Hello, world!"}],
+        api_base="https://exampleopenaiendpoint-production.up.railway.app/v1/chat/completions",
+        stream=stream,
+    )
+    print(f"response: {response}")
+    if stream:
+        for chunk in response:
+            print(chunk)
+    else:
+        assert response is not None
+
+
+@pytest.mark.parametrize("stream", [True, False])
+@pytest.mark.asyncio
+async def test_deepseek_mock_async_completion(stream):
+    """
+    Deepseek API is hanging. Mock the call, to a fake endpoint, so we can confirm our integration is working.
+    """
+    import litellm
+    from litellm import completion, acompletion
+
+    litellm._turn_on_debug()
+
+    response = await acompletion(
+        model="deepseek/deepseek-reasoner",
+        messages=[{"role": "user", "content": "Hello, world!"}],
+        api_base="https://exampleopenaiendpoint-production.up.railway.app/v1/chat/completions",
+        stream=stream,
+    )
+    print(f"response: {response}")
+    if stream:
+        async for chunk in response:
+            print(chunk)
+    else:
+        assert response is not None