Repository: LAION-AI/Open-Assistant
File: inference/server/oasst_inference_server/plugins/web_retriever/main.py (Python)

Code before:
    return limited_images


def truncate_paragraphs(paragraphs, max_length):
    truncated_paragraphs = []
    current_length = 0

    for paragraph in paragraphs:
        if current_length + len(paragraph) <= max_length:
            truncated_paragraphs.append(paragraph)
            current_length += len(paragraph)
        else:
            remaining_length = max_length - current_length
            truncated_paragraph = paragraph[:remaining_length]
            truncated_paragraphs.append(truncated_paragraph)
            break

    return truncated_paragraphs


@app.get("/get-url-content/", operation_id="getUrlContent", summary="It will return a web page's or pdf's content")
async def get_url_content(url: str = Query(..., description="url to fetch content from")) -> Response:
    try:
        buffer = io.BytesIO()
        encoding: str | None
        async with httpx.AsyncClient(follow_redirects=True) as client:
            async with client.stream("GET", url) as response:
                response.raise_for_status()  # Raise an exception for HTTP errors

                encoding = response.encoding
                if "Content-Length" in response.headers:
                    total = int(response.headers["Content-Length"])
                    if total > MAX_DOWNLOAD_SIZE:
                        error_message = (
                            f"Sorry, the file at {url} is too large.\nYou should report this message to the user!"
                        )
                        return JSONResponse(content={"error": error_message}, status_code=500)

                async for chunk in response.aiter_bytes():
                    buffer.write(chunk)
                    if buffer.tell() > MAX_DOWNLOAD_SIZE:
                        error_message = (
                            f"Sorry, the file at {url} is too large.\nYou should report this message to the user!"
                        )
                        return JSONResponse(content={"error": error_message}, status_code=500)

        content_bytes: bytes = buffer.getvalue()
        content_type: str = detect_content_type(content_bytes)
        buffer.seek(0)

        def decode_text() -> str:

Review comment (suggestion):
Is there a reason we need to use `httpx` over `aiohttp` here? The server already depends on `aiohttp` so we could reduce dependencies by using it, if possible

Code after:
        current_length += url_length

    return limited_images


def truncate_paragraphs(paragraphs, max_length):
    truncated_paragraphs = []
    current_length = 0

    for paragraph in paragraphs:
        if len(paragraph) == 0:
            continue
        paragraph = paragraph.strip()
        if current_length + len(paragraph) <= max_length:
            truncated_paragraphs.append(paragraph)
            current_length += len(paragraph)
        else:
            remaining_length = max_length - current_length
            truncated_paragraph = paragraph[:remaining_length]
            truncated_paragraphs.append(truncated_paragraph)
            break

    return truncated_paragraphs


@app.get("/get-url-content/", operation_id="getUrlContent", summary="It will return a web page's or pdf's content")
async def get_url_content(url: str = Query(..., description="url to fetch content from")) -> Response:
    try:
        buffer = io.BytesIO()
        encoding: str | None
        content_type: str | None

        async with aiohttp.ClientSession() as session:
            async with session.get(url) as response:
                response.raise_for_status()  # Raise an exception for HTTP errors
                try:
                    encoding = response.get_encoding()
                except RuntimeError:
                    encoding = None
                content_type = response.content_type

                if response.content_length is not None and response.content_length > MAX_DOWNLOAD_SIZE:
                    error_message = (
                        f"Sorry, the file at {url} is too large.\nYou should report this message to the user!"
                    )
                    return JSONResponse(content={"error": error_message}, status_code=500)

                async for chunk in response.content.iter_chunked(MAX_CHUNK_SIZE):
                    buffer.write(chunk)
                    if buffer.tell() > MAX_DOWNLOAD_SIZE:
                        error_message = (