Repository: ultrajson/ultrajson (language: python)

Issue:
A crash issue happens during fuzzing test
### What did you do?
We did a fuzzing test on ultrajson, a crash issue happened.

### What did you expect to happen?
python should not crash with any inputs

### What actually happened?
[Segmentation fault](https://github.com/baltsers/polyfuzz/blob/main/ultrajson/bug1/crash-info.txt).

```
Starting program: /root/anaconda3/bin/python ujson_poc.py input
[Thread debugging using libthread_db enabled]
Using host libthread_db library "/lib/x86_64-linux-gnu/libthread_db.so.1".

Program received signal SIGSEGV, Segmentation fault.
PyBytes_Size () at /tmp/build/80754af9/python-split_1631797238431/work/Objects/bytesobject.c:1199
1199    /tmp/build/80754af9/python-split_1631797238431/work/Objects/bytesobject.c: No such file or directory.
(gdb) bt
#0  PyBytes_Size () at /tmp/build/80754af9/python-split_1631797238431/work/Objects/bytesobject.c:1199
#1  0x00007ffff7e25e99 in ?? () from /root/anaconda3/lib/python3.9/site-packages/ujson.cpython-39-x86_64-linux-gnu.so
#2  0x00007ffff7e24b94 in ?? () from /root/anaconda3/lib/python3.9/site-packages/ujson.cpython-39-x86_64-linux-gnu.so
#3  0x00007ffff7e253cf in JSON_EncodeObject () from /root/anaconda3/lib/python3.9/site-packages/ujson.cpython-39-x86_64-linux-gnu.so
#4  0x00007ffff7e26e93 in objToJSON () from /root/anaconda3/lib/python3.9/site-packages/ujson.cpython-39-x86_64-linux-gnu.so
#5  0x00005555556c8714 in cfunction_call () at /tmp/build/80754af9/python-split_1631797238431/work/Objects/methodobject.c:543
#6  0x00005555556989ef in _PyObject_MakeTpCall () at /tmp/build/80754af9/python-split_1631797238431/work/Objects/call.c:191
#7  0x0000555555722d89 in _PyObject_VectorcallTstate (kwnames=0x0, nargsf=<optimized out>, args=0x7ffff7ed5758, callable=<optimized out>, tstate=<optimized out>)
    at /tmp/build/80754af9/python-split_1631797238431/work/Include/cpython/abstract.h:116
#8  PyObject_Vectorcall () at /tmp/build/80754af9/python-split_1631797238431/work/Include/cpython/abstract.h:127
#9  call_function (kwnames=0x0, oparg=<optimized out>, pp_stack=<synthetic pointer>, tstate=0x555555914680)
    at /tmp/build/80754af9/python-split_1631797238431/work/Python/ceval.c:5075
#10 _PyEval_EvalFrameDefault () at /tmp/build/80754af9/python-split_1631797238431/work/Python/ceval.c:3487
#11 0x00005555556d68e2 in _PyEval_EvalFrame () at /tmp/build/80754af9/python-split_1631797238431/work/Include/internal/pycore_ceval.h:40
#12 _PyEval_EvalCode () at /tmp/build/80754af9/python-split_1631797238431/work/Python/ceval.c:4327
#13 0x0000555555788bac in _PyEval_EvalCodeWithName (qualname=0x0, name=0x0, closure=0x0, kwdefs=0x0, defcount=0, defs=0x0, kwstep=2, kwcount=0, 
    kwargs=<optimized out>, kwnames=<optimized out>, argcount=<optimized out>, args=<optimized out>, locals=<optimized out>, globals=<optimized out>, 
    _co=<optimized out>) at /tmp/build/80754af9/python-split_1631797238431/work/Python/ceval.c:4359
#14 PyEval_EvalCodeEx () at /tmp/build/80754af9/python-split_1631797238431/work/Python/ceval.c:4375
#15 0x00005555556d79eb in PyEval_EvalCode (co=<optimized out>, globals=<optimized out>, locals=<optimized out>)
    at /tmp/build/80754af9/python-split_1631797238431/work/Python/ceval.c:826
#16 0x0000555555788c5b in run_eval_code_obj () at /tmp/build/80754af9/python-split_1631797238431/work/Python/pythonrun.c:1219
#17 0x00005555557bc705 in run_mod () at /tmp/build/80754af9/python-split_1631797238431/work/Python/pythonrun.c:1240
#18 0x000055555566160d in pyrun_file (fp=0x555555976b20, filename=0x7ffff6c7dc00, start=<optimized out>, globals=0x7ffff7f01f80, locals=0x7ffff7f01f80, closeit=1, 
    flags=0x7fffffffdfa8) at /tmp/build/80754af9/python-split_1631797238431/work/Python/pythonrun.c:1138
#19 0x00005555557c149f in pyrun_simple_file (flags=0x7fffffffdfa8, closeit=1, filename=0x7ffff6c7dc00, fp=0x555555976b20)
    at /tmp/build/80754af9/python-split_1631797238431/work/Python/pythonrun.c:449
#20 PyRun_SimpleFileExFlags () at /tmp/build/80754af9/python-split_1631797238431/work/Python/pythonrun.c:482
#21 0x00005555557c1c7f in pymain_run_file (cf=0x7fffffffdfa8, config=0x555555912f90) at /tmp/build/80754af9/python-split_1631797238431/work/Modules/main.c:379
#22 pymain_run_python (exitcode=0x7fffffffdfa0) at /tmp/build/80754af9/python-split_1631797238431/work/Modules/main.c:604
#23 Py_RunMain () at /tmp/build/80754af9/python-split_1631797238431/work/Modules/main.c:683
#24 0x00005555557c1d79 in Py_BytesMain () at /tmp/build/80754af9/python-split_1631797238431/work/Modules/main.c:1129
#25 0x00007ffff703fbf7 in __libc_start_main (main=0x555555669d80 <main>, argc=3, argv=0x7fffffffe198, init=<optimized out>, fini=<optimized out>, 
    rtld_fini=<optimized out>, stack_end=0x7fffffffe188) at ../csu/libc-start.c:310
#26 0x0000555555746bc3 in _start ()
```
### What versions are you using?

* OS: Ubuntu18.04
* Python: 3.9
* UltraJSON: latest (the main branch)

Please include **code** that reproduces the issue.
[Code](https://github.com/baltsers/polyfuzz/blob/main/ultrajson/bug1/ujson_poc.py) and [Input](https://github.com/baltsers/polyfuzz/blob/main/ultrajson/bug1/input),



Discussion / hints:

Fix patch:
diff --git a/lib/ultrajson.h b/lib/ultrajson.h
index c5d75b17..a0744fae 100644
--- a/lib/ultrajson.h
+++ b/lib/ultrajson.h
@@ -300,9 +300,10 @@ obj - An anonymous type representing the object
 enc - Function definitions for querying JSOBJ type
 buffer - Preallocated buffer to store result in. If NULL function allocates own buffer
 cbBuffer - Length of buffer (ignored if buffer is NULL)
+outLen - Will store the length of the encoded string
 
 Returns:
-Encoded JSON object as a null terminated char string.
+Encoded JSON object as a char string.
 
 NOTE:
 If the supplied buffer wasn't enough to hold the result the function will allocate a new buffer.
@@ -310,8 +311,10 @@ Life cycle of the provided buffer must still be handled by caller.
 
 If the return value doesn't equal the specified buffer caller must release the memory using
 JSONObjectEncoder.free or free() as specified when calling this function.
+
+If an error occurs during encoding, NULL is returned and no outLen is stored.
 */
-EXPORTFUNCTION char *JSON_EncodeObject(JSOBJ obj, JSONObjectEncoder *enc, char *buffer, size_t cbBuffer);
+EXPORTFUNCTION char *JSON_EncodeObject(JSOBJ obj, JSONObjectEncoder *enc, char *buffer, size_t cbBuffer, size_t *outLen);
 
 typedef struct __JSONObjectDecoder
 {
diff --git a/lib/ultrajsonenc.c b/lib/ultrajsonenc.c
index 3b6d5b7c..a5dfdd8e 100644
--- a/lib/ultrajsonenc.c
+++ b/lib/ultrajsonenc.c
@@ -948,7 +948,7 @@ static void encode(JSOBJ obj, JSONObjectEncoder *enc, const char *name, size_t c
   enc->level--;
 }
 
-char *JSON_EncodeObject(JSOBJ obj, JSONObjectEncoder *enc, char *_buffer, size_t _cbBuffer)
+char *JSON_EncodeObject(JSOBJ obj, JSONObjectEncoder *enc, char *_buffer, size_t _cbBuffer, size_t *_outLen)
 {
   enc->malloc = enc->malloc ? enc->malloc : malloc;
   enc->free =  enc->free ? enc->free : free;
@@ -984,12 +984,11 @@ char *JSON_EncodeObject(JSOBJ obj, JSONObjectEncoder *enc, char *_buffer, size_t
 
   encode (obj, enc, NULL, 0);
 
-  Buffer_Reserve(enc, 1);
   if (enc->errorMsg)
   {
     return NULL;
   }
-  Buffer_AppendCharUnchecked(enc, '\0');
 
+  *_outLen = enc->offset - enc->start;
   return enc->start;
 }
diff --git a/python/objToJSON.c b/python/objToJSON.c
index 7d0076d9..8b68f601 100644
--- a/python/objToJSON.c
+++ b/python/objToJSON.c
@@ -114,10 +114,16 @@ static void *PyStringToUTF8(JSOBJ _obj, JSONTypeContext *tc, void *outValue, siz
   return PyBytes_AsString(obj);
 }
 
-static void *PyUnicodeToUTF8(JSOBJ _obj, JSONTypeContext *tc, void *outValue, size_t *_outLen)
-{
+static char *PyUnicodeToUTF8Raw(JSOBJ _obj, size_t *_outLen, PyObject **pBytesObj)
+{
+  /*
+  Converts the PyUnicode object to char* whose size is stored in _outLen.
+  This conversion may require the creation of an intermediate PyBytes object.
+  In that case, the returned char* is in fact the internal buffer of that PyBytes object,
+  and when the char* buffer is no longer needed, the bytesObj must be DECREF'd.
+  */
   PyObject *obj = (PyObject *) _obj;
-  PyObject *newObj;
+
 #ifndef Py_LIMITED_API
   if (PyUnicode_IS_COMPACT_ASCII(obj))
   {
@@ -127,16 +133,20 @@ static void *PyUnicodeToUTF8(JSOBJ _obj, JSONTypeContext *tc, void *outValue, si
     return data;
   }
 #endif
-  newObj = PyUnicode_AsUTF8String(obj);
-  if(!newObj)
+
+  PyObject *bytesObj = *pBytesObj = PyUnicode_AsEncodedString (obj, NULL, "surrogatepass");
+  if (!bytesObj)
   {
     return NULL;
   }
 
-  GET_TC(tc)->newObj = newObj;
+  *_outLen = PyBytes_Size(bytesObj);
+  return PyBytes_AsString(bytesObj);
+}
 
-  *_outLen = PyBytes_Size(newObj);
-  return PyBytes_AsString(newObj);
+static void *PyUnicodeToUTF8(JSOBJ _obj, JSONTypeContext *tc, void *outValue, size_t *_outLen)
+{
+  return PyUnicodeToUTF8Raw(_obj, _outLen, &(GET_TC(tc)->newObj));
 }
 
 static void *PyRawJSONToUTF8(JSOBJ _obj, JSONTypeContext *tc, void *outValue, size_t *_outLen)
@@ -240,7 +250,7 @@ static int Dict_iterNext(JSOBJ obj, JSONTypeContext *tc)
   if (PyUnicode_Check(GET_TC(tc)->itemName))
   {
     itemNameTmp = GET_TC(tc)->itemName;
-    GET_TC(tc)->itemName = PyUnicode_AsUTF8String (GET_TC(tc)->itemName);
+    GET_TC(tc)->itemName = PyUnicode_AsEncodedString (GET_TC(tc)->itemName, NULL, "surrogatepass");
     Py_DECREF(itemNameTmp);
   }
   else
@@ -263,7 +273,7 @@ static int Dict_iterNext(JSOBJ obj, JSONTypeContext *tc)
       return -1;
     }
     itemNameTmp = GET_TC(tc)->itemName;
-    GET_TC(tc)->itemName = PyUnicode_AsUTF8String (GET_TC(tc)->itemName);
+    GET_TC(tc)->itemName = PyUnicode_AsEncodedString (GET_TC(tc)->itemName, NULL, "surrogatepass");
     Py_DECREF(itemNameTmp);
   }
   PRINTMARK();
@@ -332,7 +342,7 @@ static int SortedDict_iterNext(JSOBJ obj, JSONTypeContext *tc)
       // Subject the key to the same type restrictions and conversions as in Dict_iterGetValue.
       if (PyUnicode_Check(key))
       {
-        key = PyUnicode_AsUTF8String(key);
+        key = PyUnicode_AsEncodedString(key, NULL, "surrogatepass");
       }
       else if (!PyBytes_Check(key))
       {
@@ -342,7 +352,7 @@ static int SortedDict_iterNext(JSOBJ obj, JSONTypeContext *tc)
           goto error;
         }
         keyTmp = key;
-        key = PyUnicode_AsUTF8String(key);
+        key = PyUnicode_AsEncodedString(key, NULL, "surrogatepass");
         Py_DECREF(keyTmp);
       }
       else
@@ -674,7 +684,7 @@ static void Object_beginTypeContext (JSOBJ _obj, JSONTypeContext *tc, JSONObject
   {
     goto INVALID;
   }
-  PyObject* str = PyUnicode_AsEncodedString(objRepr, "utf-8", "strict");
+  PyObject* str = PyUnicode_AsEncodedString(objRepr, NULL, "strict");
   if (str)
   {
     PyErr_Format (PyExc_TypeError, "%s is not JSON serializable", PyBytes_AsString(str));
@@ -777,6 +787,7 @@ PyObject* objToJSON(PyObject* self, PyObject *args, PyObject *kwargs)
   PyObject *odefaultFn = NULL;
   int allowNan = -1;
   int orejectBytes = -1;
+  size_t retLen;
 
   JSONObjectEncoder encoder =
   {
@@ -860,7 +871,7 @@ PyObject* objToJSON(PyObject* self, PyObject *args, PyObject *kwargs)
                  csInf, csNan, 'e', DCONV_DECIMAL_IN_SHORTEST_LOW, DCONV_DECIMAL_IN_SHORTEST_HIGH, 0, 0);
 
   PRINTMARK();
-  ret = JSON_EncodeObject (oinput, &encoder, buffer, sizeof (buffer));
+  ret = JSON_EncodeObject (oinput, &encoder, buffer, sizeof (buffer), &retLen);
   PRINTMARK();
 
   dconv_d2s_free(&encoder.d2s);
@@ -881,7 +892,7 @@ PyObject* objToJSON(PyObject* self, PyObject *args, PyObject *kwargs)
     return NULL;
   }
 
-  newobj = PyUnicode_FromString (ret);
+  newobj = PyUnicode_DecodeUTF8(ret, retLen, "surrogatepass");
 
   if (ret != buffer)
   {


Test patch:
diff --git a/tests/test_ujson.py b/tests/test_ujson.py
index 9c40b97b..b36db54a 100644
--- a/tests/test_ujson.py
+++ b/tests/test_ujson.py
@@ -498,10 +498,18 @@ def test_decode_array_empty():
     assert [] == obj
 
 
-def test_encoding_invalid_unicode_character():
-    s = "\udc7f"
-    with pytest.raises(UnicodeEncodeError):
-        ujson.dumps(s)
+def test_encode_surrogate_characters():
+    assert ujson.dumps("\udc7f") == r'"\udc7f"'
+    out = r'{"\ud800":"\udfff"}'
+    assert ujson.dumps({"\ud800": "\udfff"}) == out
+    assert ujson.dumps({"\ud800": "\udfff"}, sort_keys=True) == out
+    o = {b"\xed\xa0\x80": b"\xed\xbf\xbf"}
+    assert ujson.dumps(o, reject_bytes=False) == out
+    assert ujson.dumps(o, reject_bytes=False, sort_keys=True) == out
+
+    out2 = '{"\ud800":"\udfff"}'
+    assert ujson.dumps({"\ud800": "\udfff"}, ensure_ascii=False) == out2
+    assert ujson.dumps({"\ud800": "\udfff"}, ensure_ascii=False, sort_keys=True) == out2
 
 
 def test_sort_keys():