Repository: run-llama/llama_index

Issue:
[Question]: Experiencing significant differences in the output from the LLamaIndex llm chat Api and OpenAI's direct api for image reasoning.
### Question Validation

- [x] I have searched both the documentation and discord for an answer.

### Question

I am testing the OpenAI's LLM( "gpt-4o") to extract all text from the image and format it as the html that resemble the original image.

I am using this single prompt:
text_prompt = '''Extract all the text from the given image and create an HTML code that resembles the original layout and formatting. All CSS must be defined inline at the element level. Do not include a 'style' tag at the beginning. Do not use relative widths (percentage-based). Assume the maximum width is 550px and calculate the widths based on that limit. Return only the HTML code without any additional explanation.'''

The result from direct OpenAI API is awesome.(as per my expectation)
But when used with the OpenAI abstraction in llama-index-llms-openai, the result gets worse.

However this issue is not there with  anthropic api as well with the llama-index-llms-anthropic ( infact great extraction of text/formatting)

I am attaching the code here to test.

**_Direct  API_**

`

    import os
    import json
    import base64
    from anthropic import Anthropic
    from openai import OpenAI
    
    from bs4 import BeautifulSoup
    from dotenv import load_dotenv
    load_dotenv() 
    
    
    def encode_image(image_path):
        with open(image_path, "rb") as image_file:
            return base64.b64encode(image_file.read()).decode('utf-8')
    
    def test_anthropic_model(model, text_prompt, image_path):
    
        anthropic_client = Anthropic(api_key=os.environ.get("ANTHROPIC_API_KEY"))
        
        messages = [{"role": "user", "content": []}]
        
        messages[0]["content"].append({"type": "text", "text": text_prompt})
        
        # Add image content
        image_content = {
            "type": "image",
            "source": {
                "type": "base64",
                "media_type": "image/jpeg" if image_path.endswith(".jpg") or image_path.endswith(".jpeg") else "image/png",
                "data": encode_image(image_path)
            }
        }
        messages[0]["content"].append(image_content)
        
        response = anthropic_client.messages.create(
            model=model,
            max_tokens=8000,
            messages=messages,
            thinking={
            "type": "enabled",
            "budget_tokens": 6000
        },
        )
    
        return {
            "model": model,
            "provider": "anthropic",
            "prompt": text_prompt,
            "response": response.content[1].text,
            "input_tokens": response.usage.input_tokens,
            "output_tokens": response.usage.output_tokens
        }
    
    def test_openai_model(model, text_prompt, image_path):
        
        openai_client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
        
        messages = [{"role": "user", "content": []}]
        
        # Add text content
        messages[0]["content"].append({"type": "text", "text": text_prompt})
        
        # Add image content
        image_content = {
            "type": "image_url",
            "image_url": {
                "url": f"data:image/jpeg;base64,{encode_image(image_path)}"
            }
        }
        messages[0]["content"].append(image_content)
        
        response = openai_client.chat.completions.create(
            model=model,
            messages=messages,
            max_completion_tokens=8000
        )
        
        return {
            "model": model,
            "provider": "openai",
            "prompt": text_prompt,
            "response": response.choices[0].message.content,
            "input_tokens": response.usage.prompt_tokens,
            "output_tokens": response.usage.completion_tokens
        }
    
    
    if __name__ == "__main__":
        text_prompt = "Extract all the text from the given image and create an HTML code that resembles the original layout and formatting. All CSS must be defined inline at the element level. Do not include a 'style' tag at the beginning. Do not use relative widths (percentage-based). Assume the maximum width is 550px and calculate the widths based on that limit. Return only the HTML code without any additional explanation."
     
        image_path = "sample_image.png" 
    
        model = "claude-3-7-sonnet-20250219"  
        result = test_anthropic_model(model, text_prompt, image_path)
        
        # model = "gpt-4o"
        # result = test_openai_model(model, text_prompt, image_path)
    
        # model = "o1"
        # result = test_openai_model(model, text_prompt, image_path)
    
        soup = BeautifulSoup(result["response"], "html.parser")
        print(soup.prettify())
    
    

`
**_Using LLamaIndex Abstraction_**

`

    from typing import Dict, List, Optional, Any
    from llama_index.core.llms import ChatMessage, ChatResponse, CompletionResponse, LLM, TextBlock, ImageBlock
    from llama_index.llms.openai import OpenAI
    from llama_index.llms.anthropic import Anthropic

    import os
    from dotenv import load_dotenv
        
    load_dotenv()
    
    class Model:
        def __init__(self, model_name: str, **kwargs):
            self.model_name = model_name
            self.kwargs = kwargs
            self.llm = self.initialize_llm()
        
        def initialize_llm(self) -> LLM:
            # OpenAI models
            if self.model_name.startswith(("gpt-", "o1")):
                return OpenAI(model=self.model_name, api_key=os.getenv("OPENAI_API_KEY"), **self.kwargs)
            
            # Anthropic models
            elif self.model_name.startswith(("claude-")):
                return Anthropic(
                    model=self.model_name, 
                    api_key=os.getenv("ANTHROPIC_API_KEY"), 
                    **self.kwargs
                    )
            
            else:
                raise ValueError(f"Unsupported model: {self.model_name}")
        
        def complete(self, prompt: str) -> str:
            response: CompletionResponse = self.llm.complete(prompt)
            return response.text
        
        def chat(self, text_prompt: str, image_path: Optional[str] = None) -> str:
            if image_path:
                chat_messages = [
                    ChatMessage(
                        role="user",
                        blocks=[
                            ImageBlock(path=image_path),
                            TextBlock(text=text_prompt),
                        ],
                    ),
                    ChatMessage(
                        role="user",
                        content="adhere to the user instruction strictly..."
                    )
                ]
            else:
                chat_messages = [
                    ChatMessage(
                        role="user",
                        content=text_prompt
                    )
                ]
    
            response: ChatResponse = self.llm.chat(chat_messages)
            
            return response.message.content
        
    
    
    if __name__ == "__main__":
    
        text_prompt = "Extract all the text from the given image and create an HTML code that resembles the original layout and formatting. All CSS must be defined inline at the element level. Do not include a 'style' tag at the beginning. Do not use relative widths (percentage-based). Assume the maximum width is 550px and calculate the widths based on that limit. Return only the HTML code without any additional explanation."
        image_path = "sample_image.png"
    
        model = Model(
            # model_name="gpt-4o",
            model_name="claude-3-7-sonnet-20250219",     
            max_tokens=8000,
    
            # temperature=1.0, # for sonnet thinking
            # thinking_dict={"type": "enabled", "budget_tokens": 6000}, # for sonnet thinking
            
            # max_completion_tokens=8000, # in case of o1
        )
    
        print(model.chat(text_prompt, image_path=image_path))

`

Here is the image i was testing with...

<img width="796" alt="Image" src="https://github.com/user-attachments/assets/9eeb390a-1d66-4ebb-8e55-356b69f83190" />

Hints from issue comments:
@dosubot: <!-- Greeting -->
Hey @brijesh24bs! How's it going? I'm here to help you with any bugs, questions, or contributions you might have.

<!-- Answer -->
I found a similar closed issue regarding differences in results between direct Azure OpenAI API calls and using the LlamaIndex interface. However, no solution was found in the comments of that issue <sup>[[1]](https://github.com/run-llama/llama_index/issues/6928)</sup>.

Regarding your question, the differences in output when using the LlamaIndex abstraction compared to direct OpenAI API calls could be due to several factors:

1. **Abstraction Layer**: LlamaIndex provides an abstraction over the OpenAI API, which might introduce differences in how requests are handled or processed. This abstraction includes additional features like retry logic, serialization, and environment management that are not present in direct API calls <sup>[[2]](https://github.com/run-llama/llama_index/blob/main/llama-index-integrations/llms/llama-index-llms-openai/tests/test_openai.py)</sup>.

2. **Image Handling**: The LlamaIndex abstraction might handle image inputs differently compared to direct API calls. This could affect how the image data is processed and subsequently how the text extraction and HTML formatting are performed. Unfortunately, specific details on how LlamaIndex handles image inputs differently were not found in the repository <sup>[[2]](https://github.com/run-llama/llama_index/blob/main/llama-index-integrations/llms/llama-index-llms-openai/tests/test_openai.py)</sup>.

3. **Default Parameters**: The LlamaIndex abstraction might use different default settings or parameters for the OpenAI models, which can affect the output. These include settings like `temperature`, `max_tokens`, and others that influence the model's behavior <sup>[[3]](https://github.com/run-llama/llama_index/blob/main/llama-index-integrations/llms/llama-index-llms-anyscale/llama_index/llms/anyscale/base.py)</sup>.

4. **Token Management**: LlamaIndex manages token limits and completion settings through its abstraction, which might differ from how you configure these settings in direct API calls. This management includes calculating maximum tokens based on prompt length and handling retries, which could lead to differences in output <sup>[[4]](https://github.com/run-llama/llama_index/blob/main/llama-index-integrations/llms/llama-index-llms-alephalpha/llama_index/llms/alephalpha/base.py)</sup>.

These factors could contribute to the differences you're experiencing in the output quality between the LlamaIndex abstraction and direct OpenAI API calls.


<!-- Dosu Comment Footer -->
*To continue talking to [Dosu](https://dosu.dev), mention @dosu.*

---
<sup>Help Dosu learn! Give it feedback: [Great Response](https://app.dosu.dev/response-feedback/42952625-5f93-420b-8d2d-ec36e9f82c06?feedback_type=great_response) | [Irrelevant Answer](https://app.dosu.dev/response-feedback/42952625-5f93-420b-8d2d-ec36e9f82c06?feedback_type=irrelevant_answer) | [Incorrect Sources](https://app.dosu.dev/response-feedback/42952625-5f93-420b-8d2d-ec36e9f82c06?feedback_type=incorrect_sources) | [Too Verbose](https://app.dosu.dev/response-feedback/42952625-5f93-420b-8d2d-ec36e9f82c06?feedback_type=too_verbose) | [Hallucination](https://app.dosu.dev/response-feedback/42952625-5f93-420b-8d2d-ec36e9f82c06?feedback_type=hallucination) | [Bug Report](https://app.dosu.dev/response-feedback/42952625-5f93-420b-8d2d-ec36e9f82c06?feedback_type=bug_report) | [Other](https://app.dosu.dev/response-feedback/42952625-5f93-420b-8d2d-ec36e9f82c06?feedback_type=other)</sup>
@logan-markewich: @brijesh24bs I think the issue is the image detail defaulting to "low" whereas openai defaults to auto
https://github.com/run-llama/llama_index/blob/f295afc6192f2b12bbcad2a0626153ed58bd1b40/llama-index-integrations/llms/llama-index-llms-openai/llama_index/llms/openai/utils.py#L309

Can you test this change?
```
ImageBlock(path=..., detail="auto")
```

@brijesh24bs: That was the issue. thanks @logan-markewich 


Solution patch:
diff --git a/llama-index-integrations/llms/llama-index-llms-openai/llama_index/llms/openai/utils.py b/llama-index-integrations/llms/llama-index-llms-openai/llama_index/llms/openai/utils.py
index c24cb00ea..84717d35c 100644
--- a/llama-index-integrations/llms/llama-index-llms-openai/llama_index/llms/openai/utils.py
+++ b/llama-index-integrations/llms/llama-index-llms-openai/llama_index/llms/openai/utils.py
@@ -296,7 +296,13 @@ def to_openai_message_dict(
         elif isinstance(block, ImageBlock):
             if block.url:
                 content.append(
-                    {"type": "image_url", "image_url": {"url": str(block.url)}}
+                    {
+                        "type": "image_url",
+                        "image_url": {
+                            "url": str(block.url),
+                            "detail": block.detail or "auto",
+                        },
+                    }
                 )
             else:
                 img_bytes = block.resolve_image(as_base64=True).read()
@@ -306,7 +312,7 @@ def to_openai_message_dict(
                         "type": "image_url",
                         "image_url": {
                             "url": f"data:{block.image_mimetype};base64,{img_str}",
-                            "detail": block.detail or "low",
+                            "detail": block.detail or "auto",
                         },
                     }
                 )
diff --git a/llama-index-integrations/llms/llama-index-llms-openai/pyproject.toml b/llama-index-integrations/llms/llama-index-llms-openai/pyproject.toml
index a723a67d2..47f3e9565 100644
--- a/llama-index-integrations/llms/llama-index-llms-openai/pyproject.toml
+++ b/llama-index-integrations/llms/llama-index-llms-openai/pyproject.toml
@@ -29,7 +29,7 @@ exclude = ["**/BUILD"]
 license = "MIT"
 name = "llama-index-llms-openai"
 readme = "README.md"
-version = "0.3.26"
+version = "0.3.27"
 
 [tool.poetry.dependencies]
 python = ">=3.9,<4.0"


Test patch:
diff --git a/llama-index-integrations/llms/llama-index-llms-openai/tests/test_openai_utils.py b/llama-index-integrations/llms/llama-index-llms-openai/tests/test_openai_utils.py
index a9a114212..4aadc9e47 100644
--- a/llama-index-integrations/llms/llama-index-llms-openai/tests/test_openai_utils.py
+++ b/llama-index-integrations/llms/llama-index-llms-openai/tests/test_openai_utils.py
@@ -263,7 +263,10 @@ def test_to_openai_message_dicts_with_content_blocks() -> None:
             {"type": "text", "text": "test question"},
             {
                 "type": "image_url",
-                "image_url": {"url": "https://example.com/image.jpg"},
+                "image_url": {
+                    "url": "https://example.com/image.jpg",
+                    "detail": "auto",
+                },
             },
         ],
     }