hey i built the endpoint where it porcess text 
import express from 'express';
import AIVoiceStreamService from '../openaiElevenLabs/AIVoiceStreamService';
import { UserMessage } from '../helpers/types';
import Conversation from '../db/schema';

const router = express.Router();
const aiVoiceService = new AIVoiceStreamService();

router.post('/stream', async (req: any, res: any) => {
  try {
    const { data, meetingId, context } = req.body;
    if (!data || !meetingId) {
      return res.status(400).json({ error: 'Invalid message or user ID' });
    }
    const userMessage: UserMessage = { role: 'user', content: data };

    let existingConversation = await Conversation.findOne({ meetingId });

    if (existingConversation) {
      existingConversation.messages.push(userMessage);
      const response: string = await aiVoiceService.generateVoiceStream(existingConversation.messages, res, JSON.stringify(context) || "");
      console.log(response)
      existingConversation.messages.push({ role: 'assistant', content: response || "" });
      await existingConversation.save();
      console.log('Conversation saved successfully!');

    } else {
      res.status(401).json({ error: 'Meeting room not Found' });
    }

  } catch (error) {
    console.error('Streaming error:', error);
    if (!res.headersSent) {
      res.status(500).json({ error: error instanceof Error ? error.message : 'Unknown error' });
    }
  }
});
"""
import OpenAI from 'openai';
import { response, Response } from 'express';
import { ChatMessage, AIVoiceStreamOptions } from '../helpers/types';
import WebSocket from 'ws';
import { system_prompt_natural } from '../helpers/constant';

class AIVoiceStreamService {
  private readonly openai: OpenAI;
  private readonly elevenlabsApiKey: string;
  private readonly DEFAULT_MODEL = 'eleven_flash_v2_5';
  private readonly DEFAULT_VOICE_ID = 'Xb7hH8MSUJpSbSDYk0k2';
  private readonly SENTENCE_BREAK_REGEX = /[.!?]\s+/;

  constructor() {
    const openaiApiKey = process.env.OPENAI_API_KEY;
    const elevenlabsApiKey = process.env.ELEVENLABS_API_KEY;
    if (!openaiApiKey) throw new Error('Missing OpenAI API Key');
    if (!elevenlabsApiKey) throw new Error('Missing ElevenLabs API Key');

    this.openai = new OpenAI({ apiKey: openaiApiKey });
    this.elevenlabsApiKey = elevenlabsApiKey;
  }

  private initializeWebSocket(voiceId: string, model: string): WebSocket {
    const wsUrl = `wss://api.elevenlabs.io/v1/text-to-speech/${voiceId}/stream-input?model_id=${model}`;
    console.log('Initializing ElevenLabs WebSocket connection:', wsUrl);
    
    try {
      console.log(this.elevenlabsApiKey)
      return new WebSocket(wsUrl, {
        headers: { "xi-api-key": this.elevenlabsApiKey }
      });
    } catch (error) {
      console.error('Failed to initialize ElevenLabs WebSocket:', error);
      throw error;
    }
  }

  private setupWebSocketHandlers(
    websocket: WebSocket,
    res: Response,
    resolve: () => void,
    reject: (error: Error) => void
  ): void {
    websocket.on('error', (error) => {
      console.error('ElevenLabs WebSocket error:', {
        error: error.message,
        name: error.name,
        stack: error.stack
      });
      this.handleError(error, res, reject);
    });

    websocket.on('message', (data) => {
      try {
        const chunk = JSON.parse(data.toString());
        if (chunk.audio) {
          const audioBuffer = Buffer.from(chunk.audio, 'base64');
          res.write(audioBuffer);
        } else if (chunk.error) {
          console.error('ElevenLabs returned error in message:', chunk.error);
        }
      } catch (error) {
        console.error('Error processing ElevenLabs audio chunk:', {
          error,
          dataPreview: data.toString().substring(0, 100)
        });
      }
    });

    websocket.on('close', (code, message) => {
      console.log('ElevenLabs WebSocket closed:', { code, message });
      res.end();
      resolve();
    });
  }

  private async processOpenAIStream(
    textStream: AsyncIterable<any>,
    websocket: WebSocket,
    textBuffer: { current: string },
    processTextBuffer: () => void
  ): Promise<string> {
    try {
      let fullResponse = '';

      for await (const chunk of textStream) {
        const content = chunk.choices[0]?.delta?.content || '';
        if (content) {
          textBuffer.current += content;
          fullResponse += content;
          processTextBuffer();
        }
      }

      // Send remaining text
      if (textBuffer.current) {
        try {
          websocket.send(JSON.stringify({ text: textBuffer.current }));
        } catch (error) {
          console.error('Error sending final text buffer to ElevenLabs:', {
            error,
            textLength: textBuffer.current.length
          });
        }
      }

      // Signal end of stream
      try {
        websocket.send(JSON.stringify({ text: "" }));
      } catch (error) {
        console.error('Error sending end of stream signal to ElevenLabs:', error);
      }
      
      return fullResponse;

    } catch (error) {
      console.error('Error in OpenAI stream:', error);
      throw error;
    }
  }

  private handleError(error: Error, res: Response, reject: (error: Error) => void): void {
    console.error('AIVoiceStreamService error:', {
      name: error.name,
      message: error.message,
      stack: error.stack
    });
    
    if (!res.headersSent) {
      res.status(500).json({ error: `Operation failed: ${error.message}` });
    }
    reject(error);
  }

  async generateVoiceStream(
    messages: ChatMessage[],
    res: Response,
    context: string,
    options: AIVoiceStreamOptions = {}
  ): Promise<string> {
    const {
      openaiModel = 'gpt-4o-mini',
      stability = 0.75,
      similarityBoost = 0.85,
      voiceId = this.DEFAULT_VOICE_ID
    } = options;

    console.log('Starting voice stream generation:', {
      voiceId,
      model: this.DEFAULT_MODEL,
      stability,
      similarityBoost,
      messageCount: messages.length
    });

    const fullMessages: any = [
      { role: "system", content: system_prompt_natural(context) },
      ...messages
    ];

    try {
      // Set streaming headers
      res.setHeader('Content-Type', 'audio/mpeg');
      res.setHeader('Transfer-Encoding', 'chunked');

      const websocket = this.initializeWebSocket(voiceId, this.DEFAULT_MODEL);
      const textBuffer = { current: '' };
      let wsReady = false;

      const textStream = await this.openai.beta.chat.completions.stream({
        model: openaiModel,
        messages: fullMessages,
        stream: true,
      });

      return new Promise<string>((resolve, reject) => {
        this.setupWebSocketHandlers(websocket, res, () => {}, reject);

        websocket.on('open', () => {
          console.log('ElevenLabs WebSocket connection established');
          wsReady = true;
          try {
            websocket.send(JSON.stringify({
              text: " ",
              voice_settings: {
                stability,
                similarity_boost: similarityBoost,
                use_speaker_boost: true,
              }
            }));
          } catch (error) {
            console.error('Error sending initial settings to ElevenLabs:', error);
          }
        });

        websocket.on("error", (error) => {
          console.error("ElevenLabs WebSocket error:", {
            error: error.message,
            name: error.name,
            stack: error.stack
          });
        });
      
        const processTextBuffer = () => {
          if (!wsReady) {
            console.log('Waiting for WebSocket to be ready...');
            return;
          }
          
          if (textBuffer.current.length === 0) return;

          const sentences = textBuffer.current.split(this.SENTENCE_BREAK_REGEX);
          
          if (sentences.length > 1) {
            const completeText = sentences.slice(0, -1).join('. ') + '. ';
            try {
              websocket.send(JSON.stringify({ text: completeText }));
            } catch (error) {
              console.error('Error sending text to ElevenLabs:', {
                error,
                textLength: completeText.length
              });
            }
            textBuffer.current = sentences[sentences.length - 1];
          }
        };

        this.processOpenAIStream(textStream, websocket, textBuffer, processTextBuffer)
          .then(response => resolve(response))
          .catch(error => this.handleError(error, res, reject));
      });
    } catch (error: any) {
      console.error('Voice stream generation error:', {
        name: error.name,
        message: error.message,
        stack: error.stack
      });
      
      if (!res.headersSent) {
        res.status(500).json({ error: `Failed to generate voice stream: ${error.message}` });
      }
      throw error;
    }
  }
}

export default AIVoiceStreamService;
"""
""
now i want to build another endpoint whihc takes in image liek in this docuemtns 
"""
Vision
======

Learn how to use vision capabilities to understand images.

Many OpenAI [models](/docs/models) have vision capabilities, meaning the models can take in images and answer questions about them. Historically, language model systems have been limited by taking in a single input modality, text.

Quickstart
----------

Images are made available to the model in two main ways: by passing a link to the image or by passing the base64 encoded image directly in the request. Images can be passed in the `user` messages.

Analyze the content of an image

```javascript
import OpenAI from "openai";
const openai = new OpenAI();

const response = await openai.chat.completions.create({
  model: "gpt-4o-mini",
  messages: [
    {
      role: "user",
      content: [
        { type: "text", text: "What's in this image?" },
        {
          type: "image_url",
          image_url: {
            "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
          },
        },
      ],
    },
  ],
});

console.log(response.choices[0]);
```

```python
from openai import OpenAI
client = OpenAI()

response = client.chat.completions.create(
    model="gpt-4o-mini",
    messages=[
        {
            "role": "user",
            "content": [
                {"type": "text", "text": "What's in this image?"},
                {
                    "type": "image_url",
                    "image_url": {
                        "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
                    },
                },
            ],
        }
    ],
    max_tokens=300,
)

print(response.choices[0])
```

```bash
curl https://api.openai.com/v1/chat/completions \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer $OPENAI_API_KEY" \
  -d '{
    "model": "gpt-4o-mini",
    "messages": [
      {
        "role": "user",
        "content": [
          {
            "type": "text",
            "text": "What is in this image?"
          },
          {
            "type": "image_url",
            "image_url": {
              "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg"
            }
          }
        ]
      }
    ],
    "max_tokens": 300
  }'
```

The model is best at answering general questions about what is present in the images. While it does understand the relationship between objects in images, it is not yet optimized to answer detailed questions about the location of certain objects in an image. For example, you can ask it what color a car is or what some ideas for dinner might be based on what is in you fridge, but if you show it an image of a room and ask it where the chair is, it may not answer the question correctly.

It is important to keep in mind the [limitations of the model](/docs/guides/vision#limitations) as you explore what use-cases visual understanding can be applied to.

[

Video understanding with vision

Learn how to use use GPT-4 with Vision to understand videos in the OpenAI Cookbook

](https://cookbook.openai.com/examples/gpt_with_vision_for_video_understanding)

Uploading Base64 encoded images
-------------------------------

If you have an image or set of images locally, you can pass those to the model in base 64 encoded format, here is an example of this in action:

```python
import base64
from openai import OpenAI

client = OpenAI()

# Function to encode the image
def encode_image(image_path):
    with open(image_path, "rb") as image_file:
        return base64.b64encode(image_file.read()).decode("utf-8")

# Path to your image
image_path = "path_to_your_image.jpg"

# Getting the base64 string
base64_image = encode_image(image_path)

response = client.chat.completions.create(
    model="gpt-4o-mini",
    messages=[
        {
            "role": "user",
            "content": [
                {
                    "type": "text",
                    "text": "What is in this image?",
                },
                {
                    "type": "image_url",
                    "image_url": {"url": f"data:image/jpeg;base64,{base64_image}"},
                },
            ],
        }
    ],
)

print(response.choices[0])
```

Multiple image inputs
---------------------

The Chat Completions API is capable of taking in and processing multiple image inputs in both base64 encoded format or as an image URL. The model will process each image and use the information from all of them to answer the question.

Multiple image inputs

```javascript
import OpenAI from "openai";
const openai = new OpenAI();

const response = await openai.chat.completions.create({
  model: "gpt-4o-mini",
  messages: [
    {
      role: "user",
      content: [
        { type: "text", text: "What are in these images? Is there any difference between them?" },
        {
          type: "image_url",
          image_url: {
            "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
          },
        },
        {
          type: "image_url",
          image_url: {
            "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
          },
        }
      ],
    },
  ],
});
console.log(response.choices[0]);
```

```python
from openai import OpenAI

client = OpenAI()
response = client.chat.completions.create(
    model="gpt-4o-mini",
    messages=[
        {
            "role": "user",
            "content": [
                {
                    "type": "text",
                    "text": "What are in these images? Is there any difference between them?",
                },
                {
                    "type": "image_url",
                    "image_url": {
                        "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
                    },
                },
                {
                    "type": "image_url",
                    "image_url": {
                        "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
                    },
                },
            ],
        }
    ],
    max_tokens=300,
)
print(response.choices[0])
```

```bash
curl https://api.openai.com/v1/chat/completions \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer $OPENAI_API_KEY" \
  -d '{
    "model": "gpt-4o-mini",
    "messages": [
      {
        "role": "user",
        "content": [
          {
            "type": "text",
            "text": "What are in these images? Is there any difference between them?"
          },
          {
            "type": "image_url",
            "image_url": {
              "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
            }
          },
          {
            "type": "image_url",
            "image_url": {
              "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
            }
          }
        ]
      }
    ],
    "max_tokens": 300
  }'
```

Here the model is shown two copies of the same image and can answer questions about both or each of the images independently.

Low or high fidelity image understanding
----------------------------------------

By controlling the `detail` parameter, which has three options, `low`, `high`, or `auto`, you have control over how the model processes the image and generates its textual understanding. By default, the model will use the `auto` setting which will look at the image input size and decide if it should use the `low` or `high` setting.

*   `low` will enable the "low res" mode. The model will receive a low-res 512px x 512px version of the image, and represent the image with a budget of 85 tokens. This allows the API to return faster responses and consume fewer input tokens for use cases that do not require high detail.
*   `high` will enable "high res" mode, which first allows the model to first see the low res image (using 85 tokens) and then creates detailed crops using 170 tokens for each 512px x 512px tile.

Choosing the detail level

```javascript
import OpenAI from "openai";
const openai = new OpenAI();

const response = await openai.chat.completions.create({
  model: "gpt-4o-mini",
  messages: [
    {
      role: "user",
      content: [
        { type: "text", text: "What's in this image?" },
        {
          type: "image_url",
          image_url: {
            "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
            "detail": "low"
          },
        },
      ],
    },
  ],
});

console.log(response.choices[0]);
```

```python
from openai import OpenAI
client = OpenAI()

response = client.chat.completions.create(
    model="gpt-4o-mini",
    messages=[
        {
            "role": "user",
            "content": [
                {"type": "text", "text": "What's in this image?"},
                {
                    "type": "image_url",
                    "image_url": {
                        "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
                        "detail": "high",
                    },
                },
            ],
        }
    ],
    max_tokens=300,
)

print(response.choices[0].message.content)
```

```bash
curl https://api.openai.com/v1/chat/completions \
  -H "Content-Type: application/json" \
  -H "Authorization: Bearer $OPENAI_API_KEY" \
  -d '{
    "model": "gpt-4o-mini",
    "messages": [
      {
        "role": "user",
        "content": [
          {
            "type": "text",
            "text": "What is in this image?"
          },
          {
            "type": "image_url",
            "image_url": {
              "url": "https://upload.wikimedia.org/wikipedia/commons/thumb/d/dd/Gfp-wisconsin-madison-the-nature-boardwalk.jpg/2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg",
              "detail": "high"
            }
          }
        ]
      }
    ],
    "max_tokens": 300
  }'
```

Managing images
---------------

The Chat Completions API, unlike the Assistants API, is not stateful. That means you have to manage the messages (including images) you pass to the model yourself. If you want to pass the same image to the model multiple times, you will have to pass the image each time you make a request to the API.

For long running conversations, we suggest passing images via URL's instead of base64. The latency of the model can also be improved by downsizing your images ahead of time to be less than the maximum size they are expected them to be. For low res mode, we expect a 512px x 512px image. For high res mode, the short side of the image should be less than 768px and the long side should be less than 2,000px.

After an image has been processed by the model, it is deleted from OpenAI servers and not retained. [We do not use data uploaded via the OpenAI API to train our models](https://openai.com/enterprise-privacy).

Limitations
-----------

While GPT-4 with vision is powerful and can be used in many situations, it is important to understand the limitations of the model. Here are some of the limitations we are aware of:

*   Medical images: The model is not suitable for interpreting specialized medical images like CT scans and shouldn't be used for medical advice.
*   Non-English: The model may not perform optimally when handling images with text of non-Latin alphabets, such as Japanese or Korean.
*   Small text: Enlarge text within the image to improve readability, but avoid cropping important details.
*   Rotation: The model may misinterpret rotated / upside-down text or images.
*   Visual elements: The model may struggle to understand graphs or text where colors or styles like solid, dashed, or dotted lines vary.
*   Spatial reasoning: The model struggles with tasks requiring precise spatial localization, such as identifying chess positions.
*   Accuracy: The model may generate incorrect descriptions or captions in certain scenarios.
*   Image shape: The model struggles with panoramic and fisheye images.
*   Metadata and resizing: The model doesn't process original file names or metadata, and images are resized before analysis, affecting their original dimensions.
*   Counting: May give approximate counts for objects in images.
*   CAPTCHAS: For safety reasons, we have implemented a system to block the submission of CAPTCHAs.

Calculating costs
-----------------

Image inputs are metered and charged in tokens, just as text inputs are. The token cost of a given image is determined by two factors: its size, and the `detail` option on each image\_url block. All images with `detail: low` cost 85 tokens each. `detail: high` images are first scaled to fit within a 2048 x 2048 square, maintaining their aspect ratio. Then, they are scaled such that the shortest side of the image is 768px long. Finally, we count how many 512px squares the image consists of. Each of those squares costs **170 tokens**. Another **85 tokens** are always added to the final total.

Here are some examples demonstrating the above.

*   A 1024 x 1024 square image in `detail: high` mode costs 765 tokens
    *   1024 is less than 2048, so there is no initial resize.
    *   The shortest side is 1024, so we scale the image down to 768 x 768.
    *   4 512px square tiles are needed to represent the image, so the final token cost is `170 * 4 + 85 = 765`.
*   A 2048 x 4096 image in `detail: high` mode costs 1105 tokens
    *   We scale down the image to 1024 x 2048 to fit within the 2048 square.
    *   The shortest side is 1024, so we further scale down to 768 x 1536.
    *   6 512px tiles are needed, so the final token cost is `170 * 6 + 85 = 1105`.
*   A 4096 x 8192 image in `detail: low` most costs 85 tokens
    *   Regardless of input size, low detail images are a fixed cost.

FAQ
---

### Can I fine-tune the image capabilities in `gpt-4`?

Vision fine-tuning is available for some models - [learn more](/docs/guides/fine-tuning#vision).

### Can I use `gpt-4` to generate images?

No, you can use `dall-e-3` to generate images and `gpt-4o`, `gpt-4o-mini` or `gpt-4-turbo` to understand images.

### What type of files can I upload?

We currently support PNG (.png), JPEG (.jpeg and .jpg), WEBP (.webp), and non-animated GIF (.gif).

### Is there a limit to the size of the image I can upload?

Yes, we restrict image uploads to 20MB per image.

### Can I delete an image I uploaded?

No, we will delete the image for you automatically after it has been processed by the model.

### Where can I learn more about the considerations of GPT-4 with Vision?

You can find details about our evaluations, preparation, and mitigation work in the [GPT-4 with Vision system card](https://openai.com/contributions/gpt-4v).

We have further implemented a system to block the submission of CAPTCHAs.

### How do rate limits for GPT-4 with Vision work?

We process images at the token level, so each image we process counts towards your tokens per minute (TPM) limit. See the calculating costs section for details on the formula used to determine token count per image.

### Can GPT-4 with Vision understand image metadata?

No, the model does not receive image metadata.

### What happens if my image is unclear?

If an image is ambiguous or unclear, the model will do its best to interpret it. However, the results may be less accurate. A good rule of thumb is that if an average human cannot see the info in an image at the resolutions used in low/high res mode, then the model cannot either.
"""
please give me a endpoint that wokrs seemlessly and best and perfect endpoint