Repository: BerriAI/litellm

Issue:
[Bug]: PDF file support does not work for OpenAI
### What happened?

This article describes how to provide PDF information with the OpenAI API: https://platform.openai.com/docs/guides/pdf-files?api-mode=chat

And it works fine using the OpenAI Python client:
```
import base64
from openai import Client

filename = "test.pdf"
with open(filename, "rb") as f:
    file_content = f.read()
    base64_content = base64.b64encode(file_content).decode('utf-8')
    file_data = f"data:application/pdf;base64,{base64_content}"
    file = {
        "type": "file",
        "file": {
            "filename": filename,
            "file_data": file_data,
        },
    }
    text = "Extract the text from the PDF file in markdown format. Don't include any other text or formatting."
    openai_client = Client()
    response = openai_client.chat.completions.create(
        model="gpt-4o-mini",
        messages=[{"role": "user", "content": [file, {"type": "text", "text": text}]}],
    )
    print(response.choices[0].message.content)
```

However, the corresponding LiteLLM code fails:
```
import base64
import litellm

filename = "temp/Get_Started_With_Smallpdf.pdf"
with open(filename, "rb") as f:
    file_content = f.read()
    base64_content = base64.b64encode(file_content).decode('utf-8')
    file_data = f"data:application/pdf;base64,{base64_content}"
    file = {
        "type": "file",
        "file": {
            "filename": filename,
            "file_data": file_data,
        },
    }
    text = "Extract the text from the PDF file in markdown format. Don't include any other text or formatting."
    response = litellm.completion(
        model="gpt-4o-mini",
        messages=[{"role": "user", "content": [file, {"type": "text", "text": text}]}],
    )
    print(response.choices[0].message.content)
```


### Relevant log output

```shell
Exception: Invalid user message={'role': 'user', 'content': [{'type': 'file', 'file': {'filename': 'temp/Get_Started_With_Smallpdf.pdf', 'file_data': 'data:application/pdf;base64,(truncated)'}}, {'type': 'text', 'text': "Extract the text from the PDF file in markdown format. Don't include any other text or formatting."}]} at index 0. Please ensure all user messages are valid OpenAI chat completion messages.
```

### Are you a ML Ops Team?

No

### What LiteLLM version are you on ?

v1.63.11

### Twitter / LinkedIn details

https://www.linkedin.com/in/are-meisfjord/

Discussion:
@mvrodrig: Hello @krrishdholakia and @ishaan-jaff! I was about to raise this same issue. I am on version v1.63.11-stable.
Doc: https://platform.openai.com/docs/guides/pdf-files#base64-encoded-files@ishaan-jaff: ack, this is a high priority issue. We will work on this 

Solution patch:
diff --git a/docs/my-website/docs/response_api.md b/docs/my-website/docs/response_api.md
index 0604a4258..1ab986550 100644
--- a/docs/my-website/docs/response_api.md
+++ b/docs/my-website/docs/response_api.md
@@ -29,7 +29,7 @@ import litellm
 
 # Non-streaming response
 response = litellm.responses(
-    model="gpt-4o",
+    model="o1-pro",
     input="Tell me a three sentence bedtime story about a unicorn.",
     max_output_tokens=100
 )
@@ -43,7 +43,7 @@ import litellm
 
 # Streaming response
 response = litellm.responses(
-    model="gpt-4o",
+    model="o1-pro",
     input="Tell me a three sentence bedtime story about a unicorn.",
     stream=True
 )
@@ -58,9 +58,9 @@ for event in response:
 First, add this to your litellm proxy config.yaml:
 ```yaml
 model_list:
-  - model_name: gpt-4o
+  - model_name: o1-pro
     litellm_params:
-      model: openai/gpt-4o
+      model: openai/o1-pro
       api_key: os.environ/OPENAI_API_KEY
 ```
 
@@ -85,7 +85,7 @@ client = OpenAI(
 
 # Non-streaming response
 response = client.responses.create(
-    model="gpt-4o",
+    model="o1-pro",
     input="Tell me a three sentence bedtime story about a unicorn."
 )
 
@@ -104,7 +104,7 @@ client = OpenAI(
 
 # Streaming response
 response = client.responses.create(
-    model="gpt-4o",
+    model="o1-pro",
     input="Tell me a three sentence bedtime story about a unicorn.",
     stream=True
 )
diff --git a/docs/my-website/sidebars.js b/docs/my-website/sidebars.js
index a92ceb3b0..1df988f37 100644
--- a/docs/my-website/sidebars.js
+++ b/docs/my-website/sidebars.js
@@ -396,6 +396,7 @@ const sidebars = {
         "observability/logfire_integration",
         "observability/argilla",
         "observability/arize_integration",
+        "observability/phoenix_integration",
         "debugging/local_debugging",
         "observability/raw_request_response",
         "observability/custom_callback",
diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py
index df746036d..4b0be9d5f 100644
--- a/litellm/types/llms/openai.py
+++ b/litellm/types/llms/openai.py
@@ -436,12 +436,24 @@ class ChatCompletionDocumentObject(TypedDict):
     citations: Optional[CitationsObject]
 
 
+class ChatCompletionFileObjectFile(TypedDict):
+    file_data: Optional[str]
+    file_id: Optional[str]
+    filename: Optional[str]
+
+
+class ChatCompletionFileObject(TypedDict):
+    type: Literal["file"]
+    file: ChatCompletionFileObjectFile
+
+
 OpenAIMessageContentListBlock = Union[
     ChatCompletionTextObject,
     ChatCompletionImageObject,
     ChatCompletionAudioObject,
     ChatCompletionDocumentObject,
     ChatCompletionVideoObject,
+    ChatCompletionFileObject,
 ]
 
 OpenAIMessageContent = Union[
@@ -523,6 +535,7 @@ ValidUserMessageContentTypes = [
     "input_audio",
     "document",
     "video_url",
+    "file",
 ]  # used for validating user messages. Prevent users from accidentally sending anthropic messages.
 
 AllMessageValues = Union[


Test patch:
diff --git a/tests/litellm_utils_tests/test_utils.py b/tests/litellm_utils_tests/test_utils.py
index 2b1e78a68..42df7c495 100644
--- a/tests/litellm_utils_tests/test_utils.py
+++ b/tests/litellm_utils_tests/test_utils.py
@@ -1094,6 +1094,26 @@ def test_is_base64_encoded_2():
     [
         ([{"role": "user", "content": "hi"}], True),
         ([{"role": "user", "content": [{"type": "text", "text": "hi"}]}], True),
+        (
+            [
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "file",
+                            "file": {
+                                "file_id": "123",
+                                "file_name": "test.txt",
+                                "file_size": 100,
+                                "file_type": "text/plain",
+                                "file_url": "https://example.com/test.txt",
+                            },
+                        }
+                    ],
+                }
+            ],
+            True,
+        ),
         (
             [
                 {