Repository: bgreni/EmberJson
Pull request #31: Simplify and make `copy_to_string` safer
Author: login: martinvuyk
Created: 2025-05-22T01:51:48Z | Merged: 2025-05-23T19:14:29Z

Simplify and make `copy_to_string` safer. I don't know if I'm being paranoid on the pointer bounds checks, I'm very uncomfortable without having the knowledge that they are valid data


Diff:
diff --git a/emberjson/_parser_helper.mojo b/emberjson/_parser_helper.mojo
index 9d78fe8..121b495 100644
--- a/emberjson/_parser_helper.mojo
+++ b/emberjson/_parser_helper.mojo
@@ -6,7 +6,7 @@ from .tables import *
 from memory.unsafe import bitcast, pack_bits, _uint
 from bit import count_trailing_zeros
 from sys.info import bitwidthof
-from sys.intrinsics import _type_is_eq
+from sys.intrinsics import _type_is_eq, likely, unlikely
 
 alias smallest_power: Int64 = -342
 alias largest_power: Int64 = 308
@@ -129,11 +129,31 @@ fn hex_to_u32(p: BytePtr) -> UInt32:
     return v.reduce_or()
 
 
-fn handle_unicode_codepoint(mut p: BytePtr, mut dest: String) raises:
+fn handle_unicode_codepoint(
+    mut p: BytePtr, mut dest: String, end: BytePtr
+) raises:
+    # TODO: is this check necessary or just being paranoid?
+    # because theoretically no string can be built with "\u" only
+    # But if this points to bytes received over the wire, it makes sense
+    # unless we use _is_valid_utf8 at the beginning of where this is called
+    if unlikely(p + 3 >= end):
+        raise Error("Bad unicode codepoint")
     var c1 = hex_to_u32(p)
     p += 4
+    # NOTE: incredibly, this is part of the JSON standard (thanks javascript...)
+    # ECMA-404 2nd Edition / December 2017. Section 9:
+    # To escape a code point that is not in the Basic Multilingual Plane, the
+    # character may be represented as a twelve-character sequence, encoding the
+    # UTF-16 surrogate pair corresponding to the code point. So for example, a
+    # string containing only the G clef character (U+1D11E) may be represented
+    # as "\uD834\uDD1E". However, whether a processor of JSON texts interprets
+    # such a surrogate pair as a single code point or as an explicit surrogate
+    # pair is a semantic decision that is determined by the specific processor.
     if c1 >= 0xD800 and c1 < 0xDC00:
-        if unlikely(p[] != `\\` and (p + 1)[] != `u`):
+        # TODO: same as the above TODO
+        if unlikely(p + 5 >= end):
+            raise Error("Bad unicode codepoint")
+        elif unlikely(not (p[0] == `\\` and p[1] == `u`)):
             raise Error("Bad unicode codepoint")
 
         p += 2
@@ -142,28 +162,25 @@ fn handle_unicode_codepoint(mut p: BytePtr, mut dest: String) raises:
         if unlikely(Bool((c1 | c2) >> 16)):
             raise Error("Bad unicode codepoint")
 
-        c1 = (((c1 - 0xD800) << 10) | (c2 - 0xDC00)) + 0x10000
+        c1 = (((c1 - 0xD800) << 10) | (c2 - 0xDC00)) | 0x10000
         p += 4
-    if c1 <= 0x7F:
+
+    if likely(c1 <= 0x7F):
         dest.append_byte(c1.cast[DType.uint8]())
-        return
     elif c1 <= 0x7FF:
-        dest.append_byte(((c1 >> 6) + 192).cast[DType.uint8]())
-        dest.append_byte(((c1 & 63) + 128).cast[DType.uint8]())
-        return
+        dest.append_byte(((c1 >> 6) | 192).cast[DType.uint8]())
+        dest.append_byte(((c1 & 63) | 128).cast[DType.uint8]())
     elif c1 <= 0xFFFF:
-        dest.append_byte(((c1 >> 12) + 224).cast[DType.uint8]())
-        dest.append_byte((((c1 >> 6) & 63) + 128).cast[DType.uint8]())
-        dest.append_byte(((c1 & 63) + 128).cast[DType.uint8]())
-        return
-    elif c1 <= 0x10FFFF:
-        dest.append_byte(((c1 >> 18) + 240).cast[DType.uint8]())
-        dest.append_byte((((c1 >> 12) & 63) + 128).cast[DType.uint8]())
-        dest.append_byte((((c1 >> 6) & 63) + 128).cast[DType.uint8]())
-        dest.append_byte(((c1 & 63) + 128).cast[DType.uint8]())
-        return
+        dest.append_byte(((c1 >> 12) | 224).cast[DType.uint8]())
+        dest.append_byte((((c1 >> 6) & 63) | 128).cast[DType.uint8]())
+        dest.append_byte(((c1 & 63) | 128).cast[DType.uint8]())
     else:
-        raise Error("Invalid unicode")
+        if unlikely(c1 > 0x10FFFF):
+            raise Error("Invalid unicode")
+        dest.append_byte(((c1 >> 18) | 240).cast[DType.uint8]())
+        dest.append_byte((((c1 >> 12) & 63) | 128).cast[DType.uint8]())
+        dest.append_byte((((c1 >> 6) & 63) | 128).cast[DType.uint8]())
+        dest.append_byte(((c1 & 63) | 128).cast[DType.uint8]())
 
 
 @always_inline
@@ -178,31 +195,26 @@ fn copy_to_string[
     fn decode_unicode(out res: String) raises:
         # This will usually slightly overallocate if the string contains
         # escaped unicode
-        var l = String(capacity=length + 1)
+        var l = String(capacity=length)
         var p = start
 
         while p < end:
-            if p[] == `\\` and p + 1 != end and (p + 1)[] == `u`:
+            if p[0] == `\\` and p[Int(p + 1 != end)] == `u`:
                 p += 2
-                handle_unicode_codepoint(p, l)
+                handle_unicode_codepoint(p, l, end)
             else:
-                l.append_byte(p[])
+                l.append_byte(p[0])
                 p += 1
         res = l^
 
-    @parameter
-    fn bulk_copy(out res: String):
-        var slice = StringSlice(ptr=start, length=length)
-        res = String(bytes=slice.as_bytes())
-
     @parameter
     if not ignore_unicode:
         if found_unicode:
             return decode_unicode()
         else:
-            return bulk_copy()
+            return String(StringSlice(ptr=start, length=length))
     else:
-        return bulk_copy()
+        return String(StringSlice(ptr=start, length=length))
 
 
 @always_inline
diff --git a/test/emberjson/test_parser.mojo b/test/emberjson/test_parser.mojo
index 8504dc6..15e9536 100644
--- a/test/emberjson/test_parser.mojo
+++ b/test/emberjson/test_parser.mojo
@@ -25,3 +25,56 @@ def test_parse():
     assert_equal(String(json), "[123,345]")
 
     assert_equal(len(json.array()), 2)
+
+
+def test_parse_utf16_surrogates():
+    var s: String = r'{"key": "To decode U+10437 (\uD801\uDC37) from UTF-16:"}'
+    var p = Parser(s)
+    var json = p.parse()
+    assert_true(json.is_object())
+    assert_true(json.object()["key"].isa[String]())
+    assert_equal(
+        json.object()["key"].string(), "To decode U+10437 (𐐷) from UTF-16:"
+    )
+
+    var s2: String = r'{"\uD801\uDC37": "\uD801\uDC37"}'
+    var p2 = Parser(s2)
+    var json2 = p2.parse()
+    assert_true(json2.is_object())
+    assert_true(json2.object()["𐐷"].isa[String]())
+    assert_equal(json2.object()["𐐷"].string(), "𐐷")
+
+
+def test_parse_wrong_backslash():
+    var data = List('{"key": "This should raise and not segfault:'.as_bytes())
+    data.append(Byte(ord("\\")))
+    with assert_raises():
+        var p = Parser(Span(data))
+        _ = p.parse()
+    data.append(Byte(ord("u")))
+    with assert_raises():
+        var p = Parser(Span(data))
+        _ = p.parse()
+    data.extend("D801".as_bytes())
+    with assert_raises():
+        var p = Parser(Span(data))
+        _ = p.parse()
+    data.append(Byte(ord("\\")))
+    with assert_raises():
+        var p = Parser(Span(data))
+        _ = p.parse()
+    data.append(Byte(ord("u")))
+    with assert_raises():
+        var p = Parser(Span(data))
+        _ = p.parse()
+    data.extend("D801".as_bytes())
+    with assert_raises():
+        var p = Parser(Span(data))
+        _ = p.parse()
+
+    var data2 = List('{"key": '.as_bytes())
+    data2.append(Byte(ord("\\")))
+    data2.extend('"this should not be correct"}'.as_bytes())
+    var p2 = Parser(Span(data2))
+    with assert_raises():
+        _ = p2.parse()


Reviews:
user: login: bgreni
state: CHANGES_REQUESTED
author_association: OWNER
submitted_at: 2025-05-22T02:35:34Z
user: login: martinvuyk
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-22T05:10:59Z
user: login: martinvuyk
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-22T05:11:55Z
user: login: martinvuyk
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-22T05:13:11Z
user: login: martinvuyk
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-22T05:14:12Z
user: login: bgreni
state: COMMENTED
author_association: OWNER
submitted_at: 2025-05-22T22:16:58Z
user: login: martinvuyk
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-22T22:26:18Z
user: login: bgreni
state: COMMENTED
author_association: OWNER
submitted_at: 2025-05-22T22:29:06Z
user: login: soraros
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-22T23:16:58Z
user: login: bgreni
state: COMMENTED
author_association: OWNER
submitted_at: 2025-05-22T23:23:03Z
user: login: bgreni
state: COMMENTED
author_association: OWNER
submitted_at: 2025-05-23T00:03:35Z
user: login: martinvuyk
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-23T00:28:38Z
user: login: bgreni
state: COMMENTED
author_association: OWNER
submitted_at: 2025-05-23T00:30:31Z
user: login: bgreni
state: COMMENTED
author_association: OWNER
submitted_at: 2025-05-23T00:31:51Z
user: login: bgreni
state: COMMENTED
author_association: OWNER
submitted_at: 2025-05-23T16:02:12Z
user: login: martinvuyk
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-23T17:03:21Z
user: login: bgreni
state: COMMENTED
author_association: OWNER
submitted_at: 2025-05-23T17:10:58Z
user: login: martinvuyk
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-23T17:12:38Z
user: login: bgreni
body: Looks good! Appreciate your numerous contributions!
state: APPROVED
author_association: OWNER
submitted_at: 2025-05-23T19:14:22Z

Review comments:
diff_hunk: @@ -442,7 +442,7 @@ struct Parser[origin: ImmutableOrigin, options: ParseOptions = ParseOptions()]:
         while parse_digit(p, i):
             p += 1
 
-        var digit_count = ptr_dist(start_digits.p, p.p)
+        var digit_count = Int(p.p) - Int(start_digits.p)
path: emberjson/parser.mojo
user: login: bgreni
body: I don't really see the point of manually inlining this function call
created_at: 2025-05-22T02:22:25Z
author_association: OWNER
diff_hunk: @@ -141,11 +136,31 @@ fn hex_to_u32(p: BytePtr) -> UInt32:
     return v.reduce_or()
 
 
-fn handle_unicode_codepoint(mut p: BytePtr, mut dest: String) raises:
+fn handle_unicode_codepoint(
+    mut p: BytePtr, mut dest: String, end: BytePtr
+) raises:
+    # TODO: is this check necessary or just being paranoid?
path: emberjson/parser_helper.mojo
user: login: bgreni
body: At least we won't pay the price of checking for strings that are exclusively ascii. But it would be nice if we could prove with some testing that they're necessary in the first place
created_at: 2025-05-22T02:24:42Z
author_association: OWNER
diff_hunk: @@ -184,37 +196,32 @@ fn copy_to_string[
 ](
     out s: String, start: BytePtr, end: BytePtr, found_unicode: Bool = True
 ) raises:
-    var length = ptr_dist(start, end)
+    var length = Int(end) - Int(start)
 
     @parameter
     fn decode_unicode(out res: String) raises:
         # This will usually slightly overallocate if the string contains
         # escaped unicode
-        var l = String(capacity=length + 1)
+        var l = String(capacity=length)
         var p = start
 
         while p < end:
-            if p[] == `\\` and p + 1 != end and (p + 1)[] == U:
+            if p[0] == `\\` and p[Int(p + 1 != end)] == U:
                 p += 2
-                handle_unicode_codepoint(p, l)
+                handle_unicode_codepoint(p, l, end)
             else:
-                l.append_byte(p[])
+                l.append_byte(p[0])
path: emberjson/parser_helper.mojo
user: login: bgreni
body: Whats the point of using `p[0]` instead of `p[]`?
created_at: 2025-05-22T02:27:38Z
author_association: OWNER
diff_hunk: @@ -184,37 +196,32 @@ fn copy_to_string[
 ](
     out s: String, start: BytePtr, end: BytePtr, found_unicode: Bool = True
 ) raises:
-    var length = ptr_dist(start, end)
+    var length = Int(end) - Int(start)
 
     @parameter
     fn decode_unicode(out res: String) raises:
         # This will usually slightly overallocate if the string contains
         # escaped unicode
-        var l = String(capacity=length + 1)
+        var l = String(capacity=length)
         var p = start
 
         while p < end:
-            if p[] == `\\` and p + 1 != end and (p + 1)[] == U:
+            if p[0] == `\\` and p[Int(p + 1 != end)] == U:
                 p += 2
-                handle_unicode_codepoint(p, l)
+                handle_unicode_codepoint(p, l, end)
             else:
-                l.append_byte(p[])
+                l.append_byte(p[0])
                 p += 1
         res = l^
 
-    @parameter
-    fn bulk_copy(out res: String):
-        var slice = StringSlice(ptr=start, length=length)
-        res = String(bytes=slice.as_bytes())
-
     @parameter
     if not ignore_unicode:
         if found_unicode:
             return decode_unicode()
         else:
-            return bulk_copy()
+            return String(StringSlice(ptr=start, length=length))
path: emberjson/parser_helper.mojo
user: login: bgreni
body: Same as the first comment
created_at: 2025-05-22T02:28:36Z
author_association: OWNER
diff_hunk: @@ -442,7 +442,7 @@ struct Parser[origin: ImmutableOrigin, options: ParseOptions = ParseOptions()]:
         while parse_digit(p, i):
             p += 1
 
-        var digit_count = ptr_dist(start_digits.p, p.p)
+        var digit_count = Int(p.p) - Int(start_digits.p)
path: emberjson/parser.mojo
user: login: martinvuyk
body: I don't see the point in having a helper function for something that takes less than half a line and also adds IMO needless headspace for something that should actually be supported by the type i.e. `p.p - start_digits.p` should return an `Int`
created_at: 2025-05-22T05:10:59Z
author_association: CONTRIBUTOR
diff_hunk: @@ -184,37 +196,32 @@ fn copy_to_string[
 ](
     out s: String, start: BytePtr, end: BytePtr, found_unicode: Bool = True
 ) raises:
-    var length = ptr_dist(start, end)
+    var length = Int(end) - Int(start)
 
     @parameter
     fn decode_unicode(out res: String) raises:
         # This will usually slightly overallocate if the string contains
         # escaped unicode
-        var l = String(capacity=length + 1)
+        var l = String(capacity=length)
         var p = start
 
         while p < end:
-            if p[] == `\\` and p + 1 != end and (p + 1)[] == U:
+            if p[0] == `\\` and p[Int(p + 1 != end)] == U:
                 p += 2
-                handle_unicode_codepoint(p, l)
+                handle_unicode_codepoint(p, l, end)
             else:
-                l.append_byte(p[])
+                l.append_byte(p[0])
path: emberjson/parser_helper.mojo
user: login: martinvuyk
body: I like thinking of offsets instead of pointer deref, just a personal preference
created_at: 2025-05-22T05:11:55Z
author_association: CONTRIBUTOR
diff_hunk: @@ -184,37 +196,32 @@ fn copy_to_string[
 ](
     out s: String, start: BytePtr, end: BytePtr, found_unicode: Bool = True
 ) raises:
-    var length = ptr_dist(start, end)
+    var length = Int(end) - Int(start)
 
     @parameter
     fn decode_unicode(out res: String) raises:
         # This will usually slightly overallocate if the string contains
         # escaped unicode
-        var l = String(capacity=length + 1)
+        var l = String(capacity=length)
         var p = start
 
         while p < end:
-            if p[] == `\\` and p + 1 != end and (p + 1)[] == U:
+            if p[0] == `\\` and p[Int(p + 1 != end)] == U:
                 p += 2
-                handle_unicode_codepoint(p, l)
+                handle_unicode_codepoint(p, l, end)
             else:
-                l.append_byte(p[])
+                l.append_byte(p[0])
                 p += 1
         res = l^
 
-    @parameter
-    fn bulk_copy(out res: String):
-        var slice = StringSlice(ptr=start, length=length)
-        res = String(bytes=slice.as_bytes())
-
     @parameter
     if not ignore_unicode:
         if found_unicode:
             return decode_unicode()
         else:
-            return bulk_copy()
+            return String(StringSlice(ptr=start, length=length))
path: emberjson/parser_helper.mojo
user: login: martinvuyk
body: the more functions you add the more it bloats compile times, and again one liners make no sense to be abstracted away when simple to read IMO
created_at: 2025-05-22T05:13:11Z
author_association: CONTRIBUTOR
diff_hunk: @@ -141,11 +136,31 @@ fn hex_to_u32(p: BytePtr) -> UInt32:
     return v.reduce_or()
 
 
-fn handle_unicode_codepoint(mut p: BytePtr, mut dest: String) raises:
+fn handle_unicode_codepoint(
+    mut p: BytePtr, mut dest: String, end: BytePtr
+) raises:
+    # TODO: is this check necessary or just being paranoid?
path: emberjson/parser_helper.mojo
user: login: martinvuyk
body: I've been thinking of proposing a lazy reader approach that pre-validates and helps with bounds checks, I'll tinker a bit and open an issue/PR for it if it works
created_at: 2025-05-22T05:14:11Z
author_association: CONTRIBUTOR
diff_hunk: @@ -442,7 +442,7 @@ struct Parser[origin: ImmutableOrigin, options: ParseOptions = ParseOptions()]:
         while parse_digit(p, i):
             p += 1
 
-        var digit_count = ptr_dist(start_digits.p, p.p)
+        var digit_count = Int(p.p) - Int(start_digits.p)
path: emberjson/parser.mojo
user: login: bgreni
body: > actually be supported by the type i.e. p.p - start_digits.p should return an Int

And if that ever happens, now we have to change a few dozen LOC instead of 1. Mojo is constantly getting broken and I'd prefer to make an effort to limit the refactoring work when they do.
created_at: 2025-05-22T22:16:58Z
author_association: OWNER
diff_hunk: @@ -442,7 +442,7 @@ struct Parser[origin: ImmutableOrigin, options: ParseOptions = ParseOptions()]:
         while parse_digit(p, i):
             p += 1
 
-        var digit_count = ptr_dist(start_digits.p, p.p)
+        var digit_count = Int(p.p) - Int(start_digits.p)
path: emberjson/parser.mojo
user: login: martinvuyk
body: making an `Int` out of a pointer will never change, I find it more distracting to abstract for such trivial ops
created_at: 2025-05-22T22:26:18Z
author_association: CONTRIBUTOR
diff_hunk: @@ -184,37 +196,32 @@ fn copy_to_string[
 ](
     out s: String, start: BytePtr, end: BytePtr, found_unicode: Bool = True
 ) raises:
-    var length = ptr_dist(start, end)
+    var length = Int(end) - Int(start)
 
     @parameter
     fn decode_unicode(out res: String) raises:
         # This will usually slightly overallocate if the string contains
         # escaped unicode
-        var l = String(capacity=length + 1)
+        var l = String(capacity=length)
         var p = start
 
         while p < end:
-            if p[] == `\\` and p + 1 != end and (p + 1)[] == U:
+            if p[0] == `\\` and p[Int(p + 1 != end)] == U:
                 p += 2
-                handle_unicode_codepoint(p, l)
+                handle_unicode_codepoint(p, l, end)
             else:
-                l.append_byte(p[])
+                l.append_byte(p[0])
                 p += 1
         res = l^
 
-    @parameter
-    fn bulk_copy(out res: String):
-        var slice = StringSlice(ptr=start, length=length)
-        res = String(bytes=slice.as_bytes())
-
     @parameter
     if not ignore_unicode:
         if found_unicode:
             return decode_unicode()
         else:
-            return bulk_copy()
+            return String(StringSlice(ptr=start, length=length))
path: emberjson/parser_helper.mojo
user: login: bgreni
body: Compile times here are bad atm because of [this](https://github.com/modular/modular/issues/4491) I don't think avoiding inlining functions will have a worthwhile impact on compile times
created_at: 2025-05-22T22:29:05Z
author_association: OWNER
diff_hunk: @@ -442,7 +442,7 @@ struct Parser[origin: ImmutableOrigin, options: ParseOptions = ParseOptions()]:
         while parse_digit(p, i):
             p += 1
 
-        var digit_count = ptr_dist(start_digits.p, p.p)
+        var digit_count = Int(p.p) - Int(start_digits.p)
path: emberjson/parser.mojo
user: login: soraros
body: I think `ptr_dist` is actually a quite nice abstraction which aligns with your own way of thinking in offsets.
created_at: 2025-05-22T23:16:58Z
author_association: CONTRIBUTOR
diff_hunk: @@ -442,7 +442,7 @@ struct Parser[origin: ImmutableOrigin, options: ParseOptions = ParseOptions()]:
         while parse_digit(p, i):
             p += 1
 
-        var digit_count = ptr_dist(start_digits.p, p.p)
+        var digit_count = Int(p.p) - Int(start_digits.p)
path: emberjson/parser.mojo
user: login: bgreni
body: I also generally still like to abide by DRY when it's convenient
created_at: 2025-05-22T23:23:03Z
author_association: OWNER
diff_hunk: @@ -184,37 +196,32 @@ fn copy_to_string[
 ](
     out s: String, start: BytePtr, end: BytePtr, found_unicode: Bool = True
 ) raises:
-    var length = ptr_dist(start, end)
+    var length = Int(end) - Int(start)
 
     @parameter
     fn decode_unicode(out res: String) raises:
         # This will usually slightly overallocate if the string contains
         # escaped unicode
-        var l = String(capacity=length + 1)
+        var l = String(capacity=length)
         var p = start
 
         while p < end:
-            if p[] == `\\` and p + 1 != end and (p + 1)[] == U:
+            if p[0] == `\\` and p[Int(p + 1 != end)] == U:
                 p += 2
-                handle_unicode_codepoint(p, l)
+                handle_unicode_codepoint(p, l, end)
             else:
-                l.append_byte(p[])
+                l.append_byte(p[0])
path: emberjson/parser_helper.mojo
user: login: bgreni
body: technically that will result in `(p + 0)[]` which may or may not get optimized away?
created_at: 2025-05-23T00:03:35Z
author_association: OWNER
diff_hunk: @@ -184,37 +196,32 @@ fn copy_to_string[
 ](
     out s: String, start: BytePtr, end: BytePtr, found_unicode: Bool = True
 ) raises:
-    var length = ptr_dist(start, end)
+    var length = Int(end) - Int(start)
 
     @parameter
     fn decode_unicode(out res: String) raises:
         # This will usually slightly overallocate if the string contains
         # escaped unicode
-        var l = String(capacity=length + 1)
+        var l = String(capacity=length)
         var p = start
 
         while p < end:
-            if p[] == `\\` and p + 1 != end and (p + 1)[] == U:
+            if p[0] == `\\` and p[Int(p + 1 != end)] == U:
                 p += 2
-                handle_unicode_codepoint(p, l)
+                handle_unicode_codepoint(p, l, end)
             else:
-                l.append_byte(p[])
+                l.append_byte(p[0])
path: emberjson/parser_helper.mojo
user: login: martinvuyk
body: The compiler should be smart enough to figure it out especially since it is a compile time constant
created_at: 2025-05-23T00:28:37Z
author_association: CONTRIBUTOR
diff_hunk: @@ -184,37 +196,32 @@ fn copy_to_string[
 ](
     out s: String, start: BytePtr, end: BytePtr, found_unicode: Bool = True
 ) raises:
-    var length = ptr_dist(start, end)
+    var length = Int(end) - Int(start)
 
     @parameter
     fn decode_unicode(out res: String) raises:
         # This will usually slightly overallocate if the string contains
         # escaped unicode
-        var l = String(capacity=length + 1)
+        var l = String(capacity=length)
         var p = start
 
         while p < end:
-            if p[] == `\\` and p + 1 != end and (p + 1)[] == U:
+            if p[0] == `\\` and p[Int(p + 1 != end)] == U:
                 p += 2
-                handle_unicode_codepoint(p, l)
+                handle_unicode_codepoint(p, l, end)
             else:
-                l.append_byte(p[])
+                l.append_byte(p[0])
path: emberjson/parser_helper.mojo
user: login: bgreni
body: It's also stylistically inconsistent with the rest of the library
created_at: 2025-05-23T00:30:31Z
author_association: OWNER
diff_hunk: @@ -298,7 +297,7 @@ fn parse_decimal(out answer: Decimal, mut p: CheckedPointer) raises:
             while p[] == ZERO_CHAR:
                 p += 1
         consume_digits()
-        answer.decimal_point = ptr_dist(p.p, first_after_period.p)
+        answer.decimal_point = Int(first_after_period.p) - Int(p.p)
path: emberjson/slow_float_parse.mojo
user: login: bgreni
body: This isn't externally exposed so it can go back to using `ptr_dist`
created_at: 2025-05-23T00:31:51Z
author_association: OWNER
diff_hunk: @@ -25,3 +25,56 @@ def test_parse():
     assert_equal(String(json), "[123,345]")
 
     assert_equal(len(json.array()), 2)
+
+
+def test_parse_utf16_surrogates():
+    var s: String = r'{"key": "To decode U+10437 (\uD801\uDC37) from UTF-16:"}'
path: test/emberjson/test_parser.mojo
user: login: bgreni
body: Out of curiosity what are we testing here since we don't currently support anything but UTF-8?
created_at: 2025-05-23T16:02:12Z
author_association: OWNER
diff_hunk: @@ -25,3 +25,56 @@ def test_parse():
     assert_equal(String(json), "[123,345]")
 
     assert_equal(len(json.array()), 2)
+
+
+def test_parse_utf16_surrogates():
+    var s: String = r'{"key": "To decode U+10437 (\uD801\uDC37) from UTF-16:"}'
path: test/emberjson/test_parser.mojo
user: login: martinvuyk
body: `r"some string"` doesn't parse any escape sequence. This would be interpreted as two separate unicode escape sequences by Mojo (currently doesn't support `\uXXXX` and `\UXXXXXXXX` escape sequences but will in the future) and Python not a UTF-16 surrogate pair like JSON allows. The data is encoded to UTF-8 but the escape sequences can be a UTF-16 surrogate pair which is absolutely nuts.
created_at: 2025-05-23T17:03:21Z
author_association: CONTRIBUTOR
diff_hunk: @@ -25,3 +25,56 @@ def test_parse():
     assert_equal(String(json), "[123,345]")
 
     assert_equal(len(json.array()), 2)
+
+
+def test_parse_utf16_surrogates():
+    var s: String = r'{"key": "To decode U+10437 (\uD801\uDC37) from UTF-16:"}'
path: test/emberjson/test_parser.mojo
user: login: bgreni
body: Ah I see now. Yeah this line in your above comment is particularly horrifying

`such a surrogate pair as a single code point or as an explicit surrogate
    # pair is a semantic decision that is determined by the specific processor.`
created_at: 2025-05-23T17:10:58Z
author_association: OWNER
diff_hunk: @@ -25,3 +25,56 @@ def test_parse():
     assert_equal(String(json), "[123,345]")
 
     assert_equal(len(json.array()), 2)
+
+
+def test_parse_utf16_surrogates():
+    var s: String = r'{"key": "To decode U+10437 (\uD801\uDC37) from UTF-16:"}'
path: test/emberjson/test_parser.mojo
user: login: martinvuyk
body: Yep, horrible decisions by our predecessors IMO
created_at: 2025-05-23T17:12:37Z
author_association: CONTRIBUTOR

PR comments:
user: login: bgreni
created_at: 2025-05-22T02:21:40Z
body: FYI If you run a piece of test code with `-D ASSERT=all` you can check to see if if it invokes any out of bounds accesses
author_association: OWNER
user: login: martinvuyk
created_at: 2025-05-23T00:02:12Z
body: I returned all of the uses of the `ptr_dist` function inside `parser.mojo` and `_parser_helper.mojo` and left our all others to not import from the `_parser_helper.mojo` module. I also inlined `bulk_copy` because IMO it makes no sense to abstract over more lines than it takes to just do the thing in 1 line.
author_association: CONTRIBUTOR
user: login: bgreni
created_at: 2025-05-23T17:12:58Z
body: Ignoring the testing errors of course, there is a legitimate error from the `package` workflow
author_association: OWNER