Repository: TeskaLabs/cysimdjson (language: python)

Issue:
convert c++ exception to python on load() and parse()
This PR is meant to allow C++ exceptions raise by *simdjson_parser::load* and *simdjson_parser::parse* to be caught from Python. This can happen despite those methods being marked *noexcept* in the original *simdjson* repo.

### Current behaviour
The current behaviour aborts on malformed JSON:
```
$ python
>>> import cysimdjson
>>> cysimdjson.JSONParser().parse(b'{"foo": null,}')
libc++abi.dylib: terminating with uncaught exception of type simdjson::simdjson_error: The JSON document has an improper structure: missing or superfluous commas, braces, missing keys, etc.
[1]    28023 abort      python
$ 
```
or on non-existent files:
```
$ python
>>> import cysimdjson
>>> cysimdjson.JSONParser().load(b'non_existent.json')
 libc++abi.dylib: terminating with uncaught exception of type simdjson::simdjson_error: Error reading the file.
[1]    28793 abort      python
$ 
```

### New behaviour

Raises *RuntimeError* instead, in both cases:
```
>>> cysimdjson.JSONParser().parse(b'{"foo": null,}')
Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "cysimdjson/cysimdjson.pyx", line 312, in cysimdjson.JSONParser.load
    cdef simdjson_element doc = self.Parser.load(path)
RuntimeError: The JSON document has an improper structure: missing or superfluous commas, braces, missing keys, etc.
>>> 
```
```
>>> cysimdjson.JSONParser().load(b'non_existent.json')
Traceback (most recent call last):
  File "<string>", line 1, in <module>
  File "cysimdjson/cysimdjson.pyx", line 312, in cysimdjson.JSONParser.load
    cdef simdjson_element doc = self.Parser.load(path)
RuntimeError: Error reading the file.
>>> 
```

There may be other code paths needing this, but I don't have time to investigate.

### Setup tested

Anaconda Python 3.7.6, MacOS.


Discussion / hints:
Feel free to re-use parts of pysimdjson, it's MIT. You can grab the error mapper from https://github.com/TkTech/pysimdjson/pull/74/files#diff-078ecf24fc2bc348b6307b7643f83932fd27f606bb390198274458dd66b83306, if you want more specific errors than just RuntimeError.
Hi @TkTech, specific exceptions would be nice, but since I can't dedicate time to the more detailed fix you proposed or to the requisite tests, I'm just going to use my fork for now so that it doesn't crash out when I use the package. Thanks!

Fix patch:
diff --git a/cysimdjson/cysimdjson.pyx b/cysimdjson/cysimdjson.pyx
index e734cf9..44e0129 100644
--- a/cysimdjson/cysimdjson.pyx
+++ b/cysimdjson/cysimdjson.pyx
@@ -11,6 +11,8 @@ cdef extern from "string_view" namespace "std":
 	cppclass string_view:
 		pass
 
+cdef extern from "pysimdjson/errors.h":
+	cdef void simdjson_error_handler()
 
 cdef extern from "jsoninter.h" namespace "simdjson_element_type":
 	cdef simdjson_element_type STRING
@@ -65,24 +67,24 @@ cdef extern from "jsoninter.h":
 	cppclass simdjson_parser:
 		simdjson_parser()
 		simdjson_parser(size_t max_capacity)
-		simdjson_element load(string) except +
-		simdjson_element parse(const char * buf, size_t len, bool realloc_if_needed) except +
+		simdjson_element load(string) except + simdjson_error_handler
+		simdjson_element parse(const char * buf, size_t len, bool realloc_if_needed) except + simdjson_error_handler
 
-	cdef int getitem_from_element(simdjson_element & element, string & key, simdjson_element & value)
-	cdef int getitem_from_array(simdjson_array & array, int key, simdjson_element & value)
+	cdef int getitem_from_element(simdjson_element & element, string & key, simdjson_element & value) except + simdjson_error_handler
+	cdef int getitem_from_array(simdjson_array & array, int key, simdjson_element & value) except + simdjson_error_handler
 
-	cdef int at_pointer_element(simdjson_element & element, string & key, simdjson_element & value)
-	cdef int at_pointer_array(simdjson_array & array, string & key, simdjson_element & value)
+	cdef int at_pointer_element(simdjson_element & element, string & key, simdjson_element & value) except + simdjson_error_handler
+	cdef int at_pointer_array(simdjson_array & array, string & key, simdjson_element & value) except + simdjson_error_handler
 
-	cdef bool compare_type(simdjson_element_type a, simdjson_element_type b)
-	cdef object to_string(simdjson_element & value, int * ok)
-	cdef object to_int64(simdjson_element & value, int * ok)
-	cdef object to_uint64(simdjson_element & value, int * ok)
-	cdef object to_double(simdjson_element & value, int * ok)
-	cdef object to_bool(simdjson_element & value, int * ok)
+	cdef bool compare_type(simdjson_element_type a, simdjson_element_type b) except + simdjson_error_handler
+	cdef object to_string(simdjson_element & value, int * ok) except + simdjson_error_handler
+	cdef object to_int64(simdjson_element & value, int * ok) except + simdjson_error_handler
+	cdef object to_uint64(simdjson_element & value, int * ok) except + simdjson_error_handler
+	cdef object to_double(simdjson_element & value, int * ok) except + simdjson_error_handler
+	cdef object to_bool(simdjson_element & value, int * ok) except + simdjson_error_handler
 
-	cdef simdjson_array to_array(simdjson_element & value, int * ok)
-	cdef simdjson_object to_object(simdjson_element & value, int * ok)
+	cdef simdjson_array to_array(simdjson_element & value, int * ok) except + simdjson_error_handler
+	cdef simdjson_object to_object(simdjson_element & value, int * ok) except + simdjson_error_handler
 
 	PyObject * string_view_to_python_string(string_view & sv)
 	string get_active_implementation()
diff --git a/cysimdjson/pysimdjson/errors.h b/cysimdjson/pysimdjson/errors.h
index df3c069..29aecda 100644
--- a/cysimdjson/pysimdjson/errors.h
+++ b/cysimdjson/pysimdjson/errors.h
@@ -5,39 +5,4 @@
 #ifndef _PY_SIMDJSON_ERRORS
 #define _PY_SIMDJSON_ERRORS
     void simdjson_error_handler();
-    template<typename T>
-    void * flatten_array(simdjson::dom::array src, size_t *size);
-
-    template<typename T>
-    void _flatten_array(T ** buffer, simdjson::dom::array src) {
-        for (simdjson::dom::element field : src) {
-            if (field.type() == simdjson::dom::element_type::ARRAY) {
-                _flatten_array<T>(buffer, field);
-            } else {
-                **buffer = (T)field;
-                (*buffer)++;
-            }
-        }
-    }
-
-    template<typename T>
-    void * flatten_array(simdjson::dom::array src, size_t *size) {
-        // This will over-allocate if the array is not flat, since array starts
-        // and ends each count as a slot on their own. However, this prevents
-        // us from having to grow whenever we find a new child array.
-        T * data = (T*)PyMem_Malloc(sizeof(T) * (src.number_of_slots() / 2));
-        T * start = data;
-        if (!data) return NULL;
-
-        try {
-            _flatten_array<T>(&start, src);
-        } catch(...) {
-            PyMem_Free(data);
-            throw;
-        }
-
-        *size = (char*)start - (char*)data;
-        // TODO: Realloc if too large
-        return (void*)data;
-    }
 #endif


Test patch:
diff --git a/test/test_benchmark.py b/test/test_benchmark.py
index 1c911eb..834545d 100644
--- a/test/test_benchmark.py
+++ b/test/test_benchmark.py
@@ -117,7 +117,7 @@ def main():
 	test_set = [
 		perftest_orjson_parser,
 		perftest_pysimdjson_parser,
-		# perftest_libpy_simdjson_parser,
+	#	perftest_libpy_simdjson_parser,
 		perftest_pythonjson_loads,
 		perftest_cysimdjson_parse,
 		perftest_cysimdjson_pad_parse,
diff --git a/utest/test_document/__init__.py b/utest/test_document/__init__.py
index e3885d2..04cfc06 100644
--- a/utest/test_document/__init__.py
+++ b/utest/test_document/__init__.py
@@ -28,3 +28,11 @@ def test_iter_01(self):
 			"key4": 40,
 			"key5": "50",
 		}, _dict)
+
+
+
+	def test_parser_failure_01(self):
+
+		parser = cysimdjson.JSONParser()
+		with self.assertRaises(ValueError) as context:
+			json_parsed = parser.parse(b"Definitively not a JSON")