Repository: jerryjliu/gpt_index
File: llama-index-core/llama_index/core/utils.py (Python)

Code before:
        except Exception:
            decoded_bytes = raw_bytes

        try:
            # Check if raw_bytes is already base64 encoded.
            # b64decode() can succeed on random binary data, so we
            # pass verify=True to make sure it's not a false positive
            decoded_bytes = base64.b64decode(raw_bytes, validate=True)
        except BinasciiError:
            # b64decode failed, leave as is
            decoded_bytes = raw_bytes

        if as_base64:
            return BytesIO(base64.b64encode(decoded_bytes))
        return BytesIO(decoded_bytes)

    elif path is not None:
        path = Path(path) if isinstance(path, str) else path
        data = path.read_bytes()
        if as_base64:
            return BytesIO(base64.b64encode(data))
        return BytesIO(data)

    elif url is not None:
        if url.startswith("data:"):
            if as_base64 and "base64," in url:
                return BytesIO(bytes(url.split(",")[1], "utf-8"))
            elif as_base64:
                return BytesIO(base64.b64decode(url.split(",")[1]))
            else:
                return BytesIO(bytes(url.split(",")[1], "utf-8"))

        headers = {
            "User-Agent": "LlamaIndex/0.0 (https://llamaindex.ai; info@llamaindex.ai) llama-index-core/0.0"
        }
        response = requests.get(url, headers=headers, timeout=(60, 60))
        response.raise_for_status()
        if as_base64:
            return BytesIO(base64.b64encode(response.content))
        return BytesIO(response.content)

    raise ValueError("No valid source provided to resolve binary data!")


Review comment (bug):
unfortunately, I think `base64` is valid encoded base 64, so this could maybe go wrong (probably very-very rare?). Could this check be a little more explicit about the location? Probably just some more comma/semi-colon splitting

https://developer.mozilla.org/en-US/docs/Web/URI/Reference/Schemes/data

Code after:
            decoded_bytes = base64.b64decode(raw_bytes)
        except Exception:
            decoded_bytes = raw_bytes

        try:
            # Check if raw_bytes is already base64 encoded.
            # b64decode() can succeed on random binary data, so we
            # pass verify=True to make sure it's not a false positive
            decoded_bytes = base64.b64decode(raw_bytes, validate=True)
        except BinasciiError:
            # b64decode failed, leave as is
            decoded_bytes = raw_bytes

        if as_base64:
            return BytesIO(base64.b64encode(decoded_bytes))
        return BytesIO(decoded_bytes)

    elif path is not None:
        path = Path(path) if isinstance(path, str) else path
        data = path.read_bytes()
        if as_base64:
            return BytesIO(base64.b64encode(data))
        return BytesIO(data)

    elif url is not None:
        parsed_url = urlparse(url)
        if parsed_url.scheme == "data":
            # Parse data URL: data:[<mediatype>][;base64],<data>
            # The path contains everything after "data:"
            data_part = parsed_url.path

            # Split on the first comma to separate metadata from data
            if "," not in data_part:
                raise ValueError("Invalid data URL format: missing comma separator")

            metadata, url_data = data_part.split(",", 1)
            is_base64_encoded = metadata.endswith(";base64")

            if is_base64_encoded:
                # Data is base64 encoded in the URL
                decoded_data = base64.b64decode(url_data)
                if as_base64:
                    # Return as base64 bytes
                    return BytesIO(base64.b64encode(decoded_data))
                else:
                    # Return decoded binary data
                    return BytesIO(decoded_data)
            else:
                # Data is not base64 encoded in the URL (URL-encoded text)
                if as_base64:
                    # Encode the text data as base64