CVE-2024-26134: Buffer Copy without Checking Size of Input ('Classic Buffer Overflow') (CWE-120)
value: cbor2 provides encoding and decoding for the Concise Binary Object Representation (CBOR) (RFC 8949) serialization format. Starting in version 5.5.1 and prior to version 5.6.2, an attacker can crash a service using cbor2 to parse a CBOR binary by sending a long enough object. Version 5.6.2 contains a patch for this issue.
CWE description: The product copies an input buffer to an output buffer without verifying that the size of the input buffer is less than the size of the output buffer, leading to a buffer overflow.
Published: 2024-02-19T23:15Z
Repository: https://github.com/agronholm/cbor2
Language: Python
Files changed: cbor2/_decoder.py, docs/versionhistory.rst, setup.py, source/decoder.c, source/tags.c, tests/test_decoder.py
Commit date: 2024-01-14 14:12:40 +0200
Commit message: Fixed MemoryError when decoding large definite strings (#204)

Also fixed a return value check in `CBORTag_hash()`.
Diff: diff --git a/cbor2/_decoder.py b/cbor2/_decoder.py
index 9145a5d..fd0e311 100644
--- a/cbor2/_decoder.py
+++ b/cbor2/_decoder.py
@@ -1,10 +1,11 @@
 from __future__ import annotations
 
 import re
 import struct
 import sys
+from codecs import getincrementaldecoder
 from collections.abc import Callable, Mapping, Sequence
 from datetime import date, datetime, timedelta, timezone
 from io import BytesIO
 from typing import TYPE_CHECKING, Any, TypeVar, cast, overload
 
@@ -29,10 +30,11 @@ if TYPE_CHECKING:
 T = TypeVar("T")
 
 timestamp_re = re.compile(
     r"^(\d{4})-(\d\d)-(\d\d)T(\d\d):(\d\d):(\d\d)" r"(?:\.(\d{1,6})\d*)?(?:Z|([+-])(\d\d):(\d\d))$"
 )
+incremental_utf8_decoder = getincrementaldecoder("utf-8")
 
 
 class CBORDecoder:
     """
     The CBORDecoder class implements a fully featured `CBOR`_ decoder with
@@ -303,12 +305,23 @@ class CBORDecoder:
                         "non-bytestring found in indefinite length bytestring"
                     )
         else:
             if length > sys.maxsize:
                 raise CBORDecodeValueError("invalid length for bytestring 0x%x" % length)
+            elif length <= 65536:
+                result = self.read(length)
+            else:
+                # Read large bytestrings 65536 (2 ** 16) bytes at a time
+                left = length
+                buffer = bytearray()
+                while left:
+                    chunk_size = min(left, 65536)
+                    buffer.extend(self.read(chunk_size))
+                    left -= chunk_size
+
+                result = bytes(buffer)
 
-            result = self.read(length)
             self._stringref_namespace_add(result, length)
 
         return self.set_shareable(result)
 
     def decode_string(self, subtype: int) -> str:
@@ -348,11 +361,23 @@ class CBORDecoder:
                     raise CBORDecodeValueError("non-string found in indefinite length string")
         else:
             if length > sys.maxsize:
                 raise CBORDecodeValueError("invalid length for string 0x%x" % length)
 
-            result = self.read(length).decode("utf-8", self._str_errors)
+            if length <= 65536:
+                result = self.read(length).decode("utf-8", self._str_errors)
+            else:
+                # Read and decode large text strings 65536 (2 ** 16) bytes at a time
+                codec = incremental_utf8_decoder(self._str_errors)
+                left = length
+                result = ""
+                while left:
+                    chunk_size = min(left, 65536)
+                    final = left <= chunk_size
+                    result += codec.decode(self.read(chunk_size), final)
+                    left -= chunk_size
+
             self._stringref_namespace_add(result, length)
 
         return self.set_shareable(result)
 
     def decode_array(self, subtype: int) -> Sequence[Any]:
diff --git a/docs/versionhistory.rst b/docs/versionhistory.rst
index 68ca711..2b3055b 100644
--- a/docs/versionhistory.rst
+++ b/docs/versionhistory.rst
@@ -6,18 +6,20 @@ Version history
 This library adheres to `Semantic Versioning <http://semver.org/>`_.
 
 **UNRELEASED**
 
 - Added the ``cbor2`` command line tool (for ``pipx run cbor2``)
+- Added support for native date encoding (bschoenmaeckers)
 - Fixed ``SystemError`` in the C extension when decoding a ``Fractional`` with a bad
   number of arguments
 - Fixed ``SystemError`` in the C extension when the decoder object hook raises an
   exception
 - Fixed a segmentation fault when decoding invalid unicode data
 - Fixed infinite recursion when trying to hash a CBOR tag whose value points to the tag
   itself
-- Added support for native date encoding (bschoenmaeckers)
+- Fixed ``MemoryError`` when maliciously constructed bytestrings or string (declared to be absurdly
+  large) are being decoded
 
 **5.5.1** (2023-11-02)
 
 - Fixed ``CBORSimpleValue`` allowing the use of reserved values (24 to 31) which resulted in
   invalid byte sequences
diff --git a/setup.py b/setup.py
index fb903c2..a25b1dc 100644
--- a/setup.py
+++ b/setup.py
@@ -55,11 +55,11 @@ if build_c_ext:
             "source/encoder.c",
             "source/decoder.c",
             "source/tags.c",
             "source/halffloat.c",
         ],
-        optional=True,
+        # optional=True,
     )
     kwargs = {"ext_modules": [_cbor2]}
 else:
     kwargs = {}
 
diff --git a/source/decoder.c b/source/decoder.c
index 6457aed..ebb4108 100644
--- a/source/decoder.c
+++ b/source/decoder.c
@@ -346,35 +346,48 @@ _CBORDecoder_get_immutable(CBORDecoderObject *self, void *closure)
 }
 
 
 // Utility functions /////////////////////////////////////////////////////////
 
-static int
-fp_read(CBORDecoderObject *self, char *buf, const Py_ssize_t size)
+static PyObject *
+fp_read_object(CBORDecoderObject *self, const Py_ssize_t size)
 {
+    PyObject *ret = NULL;
     PyObject *obj, *size_obj;
-    char *data;
-    int ret = -1;
-
     size_obj = PyLong_FromSsize_t(size);
     if (size_obj) {
         obj = PyObject_CallFunctionObjArgs(self->read, size_obj, NULL);
+        Py_DECREF(size_obj);
         if (obj) {
             assert(PyBytes_CheckExact(obj));
             if (PyBytes_GET_SIZE(obj) == (Py_ssize_t) size) {
-                data = PyBytes_AS_STRING(obj);
-                memcpy(buf, data, size);
-                ret = 0;
+                ret = obj;
             } else {
+                Py_DECREF(obj);
                 PyErr_Format(
                     _CBOR2_CBORDecodeEOF,
                     "premature end of stream (expected to read %zd bytes, "
                     "got %zd instead)", size, PyBytes_GET_SIZE(obj));
             }
-            Py_DECREF(obj);
         }
-        Py_DECREF(size_obj);
+    }
+    return ret;
+}
+
+
+static int
+fp_read(CBORDecoderObject *self, char *buf, const Py_ssize_t size)
+{
+    int ret = -1;
+    PyObject *obj = fp_read_object(self, size);
+    if (obj) {
+        char *data = PyBytes_AS_STRING(obj);
+        if (data) {
+            memcpy(buf, data, size);
+            ret = 0;
+        }
+        Py_DECREF(obj);
     }
     return ret;
 }
 
 
@@ -536,29 +549,74 @@ decode_negint(CBORDecoderObject *self, uint8_t subtype)
     return ret;
 }
 
 
 static PyObject *
-decode_definite_bytestring(CBORDecoderObject *self, Py_ssize_t length)
+decode_definite_short_bytestring(CBORDecoderObject *self, Py_ssize_t length)
 {
-    PyObject *ret = NULL;
-
-    ret = PyBytes_FromStringAndSize(NULL, length);
+    PyObject *ret = fp_read_object(self, length);
     if (!ret)
         return NULL;
-    if (fp_read(self, PyBytes_AS_STRING(ret), length) == -1) {
-        Py_DECREF(ret);
-        return NULL;
-    }
+
     if (string_namespace_add(self, ret, length) == -1) {
         Py_DECREF(ret);
         return NULL;
     }
     return ret;
 }
 
 
+static PyObject *
+decode_definite_long_bytestring(CBORDecoderObject *self, Py_ssize_t length)
+{
+    PyObject *buffer = NULL;
+    Py_ssize_t left = length;
+    while (left) {
+        Py_ssize_t chunk_length = length <= 65536 ? length : 65536;
+        PyObject *chunk = fp_read_object(self, chunk_length);
+        if (!chunk) {
+            break;
+        }
+
+        if (!PyBytes_CheckExact(chunk)) {
+            Py_DECREF(chunk);
+            break;
+        }
+
+        if (buffer) {
+            PyObject *new_buffer = PyByteArray_Concat(buffer, chunk);
+            Py_DECREF(chunk);
+            if (!new_buffer)
+                break;
+
+            if (new_buffer != buffer) {
+                Py_DECREF(buffer);
+                buffer = new_buffer;
+            }
+        } else {
+            buffer = PyByteArray_FromObject(chunk);
+            Py_DECREF(chunk);
+            if (!buffer)
+                break;
+        }
+        left -= chunk_length;
+    }
+
+    PyObject *ret = NULL;
+    if (buffer) {
+        ret = PyBytes_FromObject(buffer);
+        Py_DECREF(buffer);
+
+        if (ret && string_namespace_add(self, ret, length) == -1) {
+            Py_DECREF(ret);
+            ret = NULL;
+        }
+    }
+    return ret;
+}
+
+
 static PyObject *
 decode_indefinite_bytestrings(CBORDecoderObject *self)
 {
     PyObject *list, *ret = NULL;
     LeadByte lead;
@@ -613,13 +671,18 @@ decode_bytestring(CBORDecoderObject *self, uint8_t subtype)
                 "excessive bytestring size 0x%s", length_hex);
         return NULL;
     }
     if (indefinite)
         ret = decode_indefinite_bytestrings(self);
+    else if (length <= 65536)
+        ret = decode_definite_short_bytestring(self, (Py_ssize_t)length);
     else
-        ret = decode_definite_bytestring(self, (Py_ssize_t)length);
-    set_shareable(self, ret);
+        ret = decode_definite_long_bytestring(self, (Py_ssize_t)length);
+
+    if (ret)
+        set_shareable(self, ret);
+
     return ret;
 }
 
 
 // NOTE: It may seem redundant to repeat the definite and indefinite routines
@@ -635,35 +698,125 @@ decode_bytestring(CBORDecoderObject *self, uint8_t subtype)
 // This precludes using the indefinite bytestring decoder above as that would
 // happily ignore UTF-8 characters split across chunks.
 
 
 static PyObject *
-decode_definite_string(CBORDecoderObject *self, Py_ssize_t length)
+decode_definite_short_string(CBORDecoderObject *self, Py_ssize_t length)
 {
-    PyObject *ret = NULL;
-    char *buf;
-
-    buf = PyMem_Malloc(length);
-    if (!buf)
-        return PyErr_NoMemory();
-
-    if (fp_read(self, buf, length) == 0)
-        ret = PyUnicode_DecodeUTF8(
-                buf, length, PyBytes_AS_STRING(self->str_errors));
-    PyMem_Free(buf);
-
-    if (!ret)
+    PyObject *bytes_obj = fp_read_object(self, length);
+    if (!bytes_obj)
         return NULL;
 
-    if (string_namespace_add(self, ret, length) == -1) {
+    const char *bytes = PyBytes_AS_STRING(bytes_obj);
+    PyObject *ret = PyUnicode_FromStringAndSize(bytes, length);
+    Py_DECREF(bytes_obj);
+    if (ret && string_namespace_add(self, ret, length) == -1) {
         Py_DECREF(ret);
         return NULL;
     }
     return ret;
 }
 
 
+static PyObject *
+decode_definite_long_string(CBORDecoderObject *self, Py_ssize_t length)
+{
+    PyObject *ret = NULL, *chunk = NULL, *string = NULL;
+    Py_ssize_t left = length;
+    Py_ssize_t consumed;
+    Py_ssize_t buffer_size = 0;  // how many bytes are allocated for the buffer
+    Py_ssize_t buffer_length = 0;  // how many bytes are actually stored in the buffer
+    char *buffer = NULL;
+    while (left) {
+        // Read up to 65536 bytes of data from the stream
+        Py_ssize_t chunk_length = 65536 - buffer_size;
+        if (left < chunk_length)
+            chunk_length = left;
+
+        PyObject *chunk = fp_read_object(self, chunk_length);
+        left -= chunk_length;
+        if (!chunk)
+            goto error;
+
+        // Get the internal buffer of the bytes object
+        char *bytes_buffer = PyBytes_AsString(chunk);
+        if (!bytes_buffer)
+            goto error;
+
+        char *source_buffer;
+        if (buffer) {
+            // Grow the buffer to accommodate the previous data plus the new chunk
+            if (buffer_length + chunk_length > buffer_size) {
+                buffer_size = buffer_length + chunk_length;
+                char *new_buffer = PyMem_Realloc(buffer, buffer_size);
+                if (!new_buffer)
+                    goto error;
+
+                buffer = new_buffer;
+            }
+
+            // Concatenate the chunk into the buffer
+            memcpy(buffer + buffer_length, bytes_buffer, chunk_length);
+            buffer_length += chunk_length;
+
+            source_buffer = buffer;
+            chunk_length = buffer_length;
+        } else {
+            // Use the chunk's internal buffer directly to decode as many characters as possible
+            source_buffer = bytes_buffer;
+        }
+
+        string = PyUnicode_DecodeUTF8Stateful(source_buffer, chunk_length, NULL, &consumed);
+        if (!string)
+            goto error;
+
+        if (ret) {
+            // Concatenate the result to the existing result
+            PyObject *joined = PyUnicode_Concat(ret, string);
+            if (!joined)
+                goto error;
+
+            Py_DECREF(string);
+            string = NULL;
+            ret = joined;
+        } else {
+            // Set the result to the decoded string
+            ret = string;
+        }
+
+        Py_ssize_t unconsumed = chunk_length - consumed;
+        if (consumed != chunk_length) {
+            if (buffer) {
+                // Move the unconsumed bytes to the start of the buffer
+                memmove(buffer, buffer + consumed, unconsumed);
+            } else {
+                // Create a new buffer
+                buffer = PyMem_Malloc(unconsumed);
+                if (!buffer)
+                    goto error;
+
+                memcpy(buffer, bytes_buffer + consumed, unconsumed);
+            }
+            buffer_length = unconsumed;
+        }
+    }
+
+    if (ret && string_namespace_add(self, ret, length) == -1)
+        goto error;
+
+    return ret;
+error:
+    Py_XDECREF(ret);
+    Py_XDECREF(chunk);
+    Py_XDECREF(string);
+    if (buffer)
+        PyMem_Free(buffer);
+
+    return NULL;
+}
+
+
 static PyObject *
 decode_indefinite_strings(CBORDecoderObject *self)
 {
     PyObject *list, *ret = NULL;
     LeadByte lead;
@@ -717,13 +870,18 @@ decode_string(CBORDecoderObject *self, uint8_t subtype)
                 "excessive string size 0x%s", length_hex);
         return NULL;
     }
     if (indefinite)
         ret = decode_indefinite_strings(self);
+    else if (length <= 65536)
+        ret = decode_definite_short_string(self, (Py_ssize_t)length);
     else
-        ret = decode_definite_string(self, (Py_ssize_t)length);
-    set_shareable(self, ret);
+        ret = decode_definite_long_string(self, (Py_ssize_t)length);
+
+    if (ret)
+        set_shareable(self, ret);
+
     return ret;
 }
 
 
 static PyObject *
diff --git a/source/tags.c b/source/tags.c
index 5cc8ea4..b718f55 100644
--- a/source/tags.c
+++ b/source/tags.c
@@ -189,11 +189,11 @@ CBORTag_hash(CBORTagObject *self)
         goto exit;
     }
 
     // Check how many more references there are in running_hashes
     Py_ssize_t length = PySequence_Length(running_hashes);
-    if (length == 1) {
+    if (length == -1) {
         ret = -1;
         goto exit;
     }
 
     // If this was the last reference, delete running_hashes from the thread-local variable
diff --git a/tests/test_decoder.py b/tests/test_decoder.py
index f23125a..dca6833 100644
--- a/tests/test_decoder.py
+++ b/tests/test_decoder.py
@@ -1,5 +1,7 @@
+from __future__ import annotations
+
 import math
 import re
 import struct
 import sys
 from binascii import unhexlify
@@ -7,10 +9,12 @@ from datetime import date, datetime, timedelta, timezone
 from decimal import Decimal
 from email.message import Message
 from fractions import Fraction
 from io import BytesIO
 from ipaddress import ip_address, ip_network
+from pathlib import Path
+from typing import Type, cast
 from uuid import UUID
 
 import pytest
 
 from cbor2 import FrozenDict
@@ -224,10 +228,11 @@ def test_binary(impl, payload, expected):
         ("6161", "a"),
         ("6449455446", "IETF"),
         ("62225c", '"\\'),
         ("62c3bc", "\u00fc"),
         ("63e6b0b4", "\u6c34"),
+        pytest.param("7a00010001" + "61" * 65535 + "c3b6", "a" * 65535 + "ö", id="split_unicode"),
     ],
 )
 def test_string(impl, payload, expected):
     decoded = impl.loads(unhexlify(payload))
     assert decoded == expected
@@ -844,22 +849,48 @@ def test_decimal_payload_unpacking(impl, data, expected):
 
 @pytest.mark.parametrize(
     "payload, exception, pattern",
     [
         pytest.param(
-            b"\xd8\x1e\x84\xff\xff\xff\xff",
+            unhexlify("d81e84ffffffff"),
             TypeError,
             r"__new__\(\) takes from 1 to 3 positional arguments but 5 were given",
             id="fractional",
         ),
         pytest.param(
-            b"\xae\xae\xae\xae\xae\xae\xae\xae\xae\x01\x08\xc2\x98C\xd9\x01\x00\xd8$"
-            b"\x9f\x00\x00\xae\xae\xff\xc2l\xa7\x99",
-            Exception,
+            unhexlify("aeaeaeaeaeaeaeaeae0108c29843d90100d8249f0000aeaeffc26ca799"),
+            "CBORDecodeEOF",
             "premature end of stream",
             id="unicode",
         ),
     ],
 )
-def test_invalid_data(impl, payload, exception, pattern) -> None:
+def test_invalid_data(
+    impl, payload: bytes, exception: type[Exception] | str, pattern: str
+) -> None:
+    if isinstance(exception, str):
+        exception = getattr(impl, exception)
+
     with pytest.raises(exception, match=pattern):
         impl.loads(payload)
+
+
+@pytest.mark.parametrize(
+    "payload",
+    [
+        pytest.param(
+            unhexlify("5b7fffffffffffff00"),
+            id="bytestring",
+        ),
+        pytest.param(
+            unhexlify("7b7fffffffffffff00"),
+            id="unicode",
+        ),
+    ],
+)
+def test_oversized_read(impl, payload: bytes, tmp_path: Path) -> None:
+    CBORDecodeEOF = cast(Type[Exception], getattr(impl, "CBORDecodeEOF"))
+    with pytest.raises(CBORDecodeEOF, match="premature end of stream"):
+        dummy_path = tmp_path / "testdata"
+        dummy_path.write_bytes(payload)
+        with dummy_path.open("rb") as f:
+            impl.load(f)

Vulnerable code:             result = self.read(length)
            result = self.read(length).decode("utf-8", self._str_errors)
- Added support for native date encoding (bschoenmaeckers)
        optional=True,
static int
fp_read(CBORDecoderObject *self, char *buf, const Py_ssize_t size)
    char *data;
    int ret = -1;

                data = PyBytes_AS_STRING(obj);
                memcpy(buf, data, size);
                ret = 0;
            Py_DECREF(obj);
        Py_DECREF(size_obj);
decode_definite_bytestring(CBORDecoderObject *self, Py_ssize_t length)
    PyObject *ret = NULL;

    ret = PyBytes_FromStringAndSize(NULL, length);
    if (fp_read(self, PyBytes_AS_STRING(ret), length) == -1) {
        Py_DECREF(ret);
        return NULL;
    }
        ret = decode_definite_bytestring(self, (Py_ssize_t)length);
    set_shareable(self, ret);
decode_definite_string(CBORDecoderObject *self, Py_ssize_t length)
    PyObject *ret = NULL;
    char *buf;

    buf = PyMem_Malloc(length);
    if (!buf)
        return PyErr_NoMemory();

    if (fp_read(self, buf, length) == 0)
        ret = PyUnicode_DecodeUTF8(
                buf, length, PyBytes_AS_STRING(self->str_errors));
    PyMem_Free(buf);

    if (!ret)
    if (string_namespace_add(self, ret, length) == -1) {
        ret = decode_definite_string(self, (Py_ssize_t)length);
    set_shareable(self, ret);
    if (length == 1) {
            b"\xd8\x1e\x84\xff\xff\xff\xff",
            b"\xae\xae\xae\xae\xae\xae\xae\xae\xae\x01\x08\xc2\x98C\xd9\x01\x00\xd8$"
            b"\x9f\x00\x00\xae\xae\xff\xc2l\xa7\x99",
            Exception,
def test_invalid_data(impl, payload, exception, pattern) -> None:
Fixed code: from codecs import getincrementaldecoder
incremental_utf8_decoder = getincrementaldecoder("utf-8")
            elif length <= 65536:
                result = self.read(length)
            else:
                # Read large bytestrings 65536 (2 ** 16) bytes at a time
                left = length
                buffer = bytearray()
                while left:
                    chunk_size = min(left, 65536)
                    buffer.extend(self.read(chunk_size))
                    left -= chunk_size

                result = bytes(buffer)
            if length <= 65536:
                result = self.read(length).decode("utf-8", self._str_errors)
            else:
                # Read and decode large text strings 65536 (2 ** 16) bytes at a time
                codec = incremental_utf8_decoder(self._str_errors)
                left = length
                result = ""
                while left:
                    chunk_size = min(left, 65536)
                    final = left <= chunk_size
                    result += codec.decode(self.read(chunk_size), final)
                    left -= chunk_size

- Added support for native date encoding (bschoenmaeckers)
- Fixed ``MemoryError`` when maliciously constructed bytestrings or string (declared to be absurdly
  large) are being decoded
        # optional=True,
static PyObject *
fp_read_object(CBORDecoderObject *self, const Py_ssize_t size)
    PyObject *ret = NULL;
        Py_DECREF(size_obj);
                ret = obj;
                Py_DECREF(obj);
    }
    return ret;
}


static int
fp_read(CBORDecoderObject *self, char *buf, const Py_ssize_t size)
{
    int ret = -1;
    PyObject *obj = fp_read_object(self, size);
    if (obj) {
        char *data = PyBytes_AS_STRING(obj);
        if (data) {
            memcpy(buf, data, size);
            ret = 0;
        }
        Py_DECREF(obj);
decode_definite_short_bytestring(CBORDecoderObject *self, Py_ssize_t length)
    PyObject *ret = fp_read_object(self, length);

static PyObject *
decode_definite_long_bytestring(CBORDecoderObject *self, Py_ssize_t length)
{
    PyObject *buffer = NULL;
    Py_ssize_t left = length;
    while (left) {
        Py_ssize_t chunk_length = length <= 65536 ? length : 65536;
        PyObject *chunk = fp_read_object(self, chunk_length);
        if (!chunk) {
            break;
        }

        if (!PyBytes_CheckExact(chunk)) {
            Py_DECREF(chunk);
            break;
        }

        if (buffer) {
            PyObject *new_buffer = PyByteArray_Concat(buffer, chunk);
            Py_DECREF(chunk);
            if (!new_buffer)
                break;

            if (new_buffer != buffer) {
                Py_DECREF(buffer);
                buffer = new_buffer;
            }
        } else {
            buffer = PyByteArray_FromObject(chunk);
            Py_DECREF(chunk);
            if (!buffer)
                break;
        }
        left -= chunk_length;
    }

    PyObject *ret = NULL;
    if (buffer) {
        ret = PyBytes_FromObject(buffer);
        Py_DECREF(buffer);

        if (ret && string_namespace_add(self, ret, length) == -1) {
            Py_DECREF(ret);
            ret = NULL;
        }
    }
    return ret;
}


    else if (length <= 65536)
        ret = decode_definite_short_bytestring(self, (Py_ssize_t)length);
        ret = decode_definite_long_bytestring(self, (Py_ssize_t)length);

    if (ret)
        set_shareable(self, ret);

decode_definite_short_string(CBORDecoderObject *self, Py_ssize_t length)
    PyObject *bytes_obj = fp_read_object(self, length);
    if (!bytes_obj)
    const char *bytes = PyBytes_AS_STRING(bytes_obj);
    PyObject *ret = PyUnicode_FromStringAndSize(bytes, length);
    Py_DECREF(bytes_obj);
    if (ret && string_namespace_add(self, ret, length) == -1) {
static PyObject *
decode_definite_long_string(CBORDecoderObject *self, Py_ssize_t length)
{
    PyObject *ret = NULL, *chunk = NULL, *string = NULL;
    Py_ssize_t left = length;
    Py_ssize_t consumed;
    Py_ssize_t buffer_size = 0;  // how many bytes are allocated for the buffer
    Py_ssize_t buffer_length = 0;  // how many bytes are actually stored in the buffer
    char *buffer = NULL;
    while (left) {
        // Read up to 65536 bytes of data from the stream
        Py_ssize_t chunk_length = 65536 - buffer_size;
        if (left < chunk_length)
            chunk_length = left;

        PyObject *chunk = fp_read_object(self, chunk_length);
        left -= chunk_length;
        if (!chunk)
            goto error;

        // Get the internal buffer of the bytes object
        char *bytes_buffer = PyBytes_AsString(chunk);
        if (!bytes_buffer)
            goto error;

        char *source_buffer;
        if (buffer) {
            // Grow the buffer to accommodate the previous data plus the new chunk
            if (buffer_length + chunk_length > buffer_size) {
                buffer_size = buffer_length + chunk_length;
                char *new_buffer = PyMem_Realloc(buffer, buffer_size);
                if (!new_buffer)
                    goto error;

                buffer = new_buffer;
            }

            // Concatenate the chunk into the buffer
            memcpy(buffer + buffer_length, bytes_buffer, chunk_length);
            buffer_length += chunk_length;

            source_buffer = buffer;
            chunk_length = buffer_length;
        } else {
            // Use the chunk's internal buffer directly to decode as many characters as possible
            source_buffer = bytes_buffer;
        }

        string = PyUnicode_DecodeUTF8Stateful(source_buffer, chunk_length, NULL, &consumed);
        if (!string)
            goto error;

        if (ret) {
            // Concatenate the result to the existing result
            PyObject *joined = PyUnicode_Concat(ret, string);
            if (!joined)
                goto error;

            Py_DECREF(string);
            string = NULL;
            ret = joined;
        } else {
            // Set the result to the decoded string
            ret = string;
        }

        Py_ssize_t unconsumed = chunk_length - consumed;
        if (consumed != chunk_length) {
            if (buffer) {
                // Move the unconsumed bytes to the start of the buffer
                memmove(buffer, buffer + consumed, unconsumed);
            } else {
                // Create a new buffer
                buffer = PyMem_Malloc(unconsumed);
                if (!buffer)
                    goto error;

                memcpy(buffer, bytes_buffer + consumed, unconsumed);
            }
            buffer_length = unconsumed;
        }
    }

    if (ret && string_namespace_add(self, ret, length) == -1)
        goto error;

    return ret;
error:
    Py_XDECREF(ret);
    Py_XDECREF(chunk);
    Py_XDECREF(string);
    if (buffer)
        PyMem_Free(buffer);

    return NULL;
}


    else if (length <= 65536)
        ret = decode_definite_short_string(self, (Py_ssize_t)length);
        ret = decode_definite_long_string(self, (Py_ssize_t)length);

    if (ret)
        set_shareable(self, ret);

    if (length == -1) {
from __future__ import annotations

from pathlib import Path
from typing import Type, cast
        pytest.param("7a00010001" + "61" * 65535 + "c3b6", "a" * 65535 + "ö", id="split_unicode"),
            unhexlify("d81e84ffffffff"),
            unhexlify("aeaeaeaeaeaeaeaeae0108c29843d90100d8249f0000aeaeffc26ca799"),
            "CBORDecodeEOF",
def test_invalid_data(
    impl, payload: bytes, exception: type[Exception] | str, pattern: str
) -> None:
    if isinstance(exception, str):
        exception = getattr(impl, exception)



@pytest.mark.parametrize(
    "payload",
    [
        pytest.param(
            unhexlify("5b7fffffffffffff00"),
            id="bytestring",
        ),
        pytest.param(
            unhexlify("7b7fffffffffffff00"),
            id="unicode",
        ),
    ],
)
def test_oversized_read(impl, payload: bytes, tmp_path: Path) -> None:
    CBORDecodeEOF = cast(Type[Exception], getattr(impl, "CBORDecodeEOF"))
    with pytest.raises(CBORDecodeEOF, match="premature end of stream"):
        dummy_path = tmp_path / "testdata"
        dummy_path.write_bytes(payload)
        with dummy_path.open("rb") as f:
            impl.load(f)