Repository: jsonata-js/jsonata (js)

Issue:
$length and other string functions return misleading results for UTF-16 surrogate pairs
If you request the length of the smiley face emoji, JSONata returns 2 instead of 1.

    $length("😂")  // returns 2

The same thing happens with any of the more exotic Unicode symbols (emoji, obscure mathematical and musical notation, anything outside the [BMP](https://en.wikipedia.org/wiki/Plane_(Unicode)#Basic_Multilingual_Plane) basically).

This is because JavaScript uses UTF-16 under the hood and needs two characters (known as a [surrogate pair](https://en.wikipedia.org/wiki/UTF-16#U+10000_to_U+10FFFF)) to represent non-BMP symbols. The equivalent JavaScript code also returns 2.

    "😂".length;  // also returns 2

This is technically correct -- surrogate pairs _do_ have two characters. But it's not very intuitive. It's also dependent on implementation language. I'm working on a Go implementation of JSONata and it doesn't have this issue (because Go uses UTF-8).

Some other expressions that return results you might not expect:

    $substring("😂", 1)  // returns an invalid character instead of an empty string

    $substring("😂", 0, 1)  // returns an invalid character instead of the emoji

    $pad("😂", 2)  // returns the emoji unpadded

    $split("😂", "")  // returns an array of two invalid characters

Note: all of these examples were run in JSONata Exerciser. I'm not sure what version of JSONata that uses.

Pull request description:
handle surrogate pairs as single char in string functions
Codepoints outside the Basic Multilingual Plane are represented in Javascript by 2 characters (surrogate pair).  This causes `$length('💩')` to return `2` rather than the expected `1`.

`$length`, `$substring` and `$pad` have been fixed to produce the expected results for strings that contain these codepoints.

Resolves #156 

Patch:
diff --git a/jsonata.js b/jsonata.js
index d4da49f..8c33752 100644
--- a/jsonata.js
+++ b/jsonata.js
@@ -3098,7 +3098,13 @@ var jsonata = (function() {
             return undefined;
         }
 
-        return str.substr(start, length);
+        var strArray = Array.from(str);
+        if(typeof length !== 'undefined') {
+            var end = start >= 0 ? start + length : strArray.length - start + length;
+            return strArray.slice(start, end).join('');
+        }
+
+        return strArray.slice(start).join('');
     }
 
     /**
@@ -3180,7 +3186,7 @@ var jsonata = (function() {
             return undefined;
         }
 
-        return str.length;
+        return Array.from(str).length;
     }
 
     /**
@@ -3225,11 +3231,11 @@ var jsonata = (function() {
         }
 
         var result;
-        var padLength = Math.abs(width) - str.length;
+        var padLength = Math.abs(width) - functionLength(str);
         if(padLength > 0) {
             var padding = (new Array(padLength + 1)).join(char);
             if(char.length > 1) {
-                padding = padding.substring(0, padLength);
+                padding = functionSubstring(padding, 0, padLength);
             }
             if(width > 0) {
                 result = str + padding;


Test patch:
diff --git a/test/test-suite/groups/function-length/case004.json b/test/test-suite/groups/function-length/case004.json
index 13d616f..0ee707c 100644
--- a/test/test-suite/groups/function-length/case004.json
+++ b/test/test-suite/groups/function-length/case004.json
@@ -2,5 +2,5 @@
     "expr": "$length(\"\\uD834\\uDD1E\")",
     "dataset": null,
     "bindings": {},
-    "result": 2
+    "result": 1
 }
\ No newline at end of file
diff --git a/test/test-suite/groups/function-length/case005.json b/test/test-suite/groups/function-length/case005.json
index cd47df8..66c6ddb 100644
--- a/test/test-suite/groups/function-length/case005.json
+++ b/test/test-suite/groups/function-length/case005.json
@@ -2,5 +2,5 @@
     "expr": "$length(\"𝄞\")",
     "dataset": null,
     "bindings": {},
-    "result": 2
+    "result": 1
 }
\ No newline at end of file
diff --git a/test/test-suite/groups/function-length/case016.json b/test/test-suite/groups/function-length/case016.json
new file mode 100644
index 0000000..578beca
--- /dev/null
+++ b/test/test-suite/groups/function-length/case016.json
@@ -0,0 +1,6 @@
+{
+    "expr": "$length(\"𝄞💩\")",
+    "dataset": null,
+    "bindings": {},
+    "result": 2
+}
\ No newline at end of file
diff --git a/test/test-suite/groups/function-pad/case007.json b/test/test-suite/groups/function-pad/case007.json
new file mode 100644
index 0000000..ecb3f4a
--- /dev/null
+++ b/test/test-suite/groups/function-pad/case007.json
@@ -0,0 +1,6 @@
+{
+    "expr": "$pad('a𝄞💩', 7)",
+    "dataset": null,
+    "bindings": {},
+    "result": "a𝄞💩    "
+}
\ No newline at end of file
diff --git a/test/test-suite/groups/function-pad/case008.json b/test/test-suite/groups/function-pad/case008.json
new file mode 100644
index 0000000..7a96939
--- /dev/null
+++ b/test/test-suite/groups/function-pad/case008.json
@@ -0,0 +1,6 @@
+{
+    "expr": "$pad('𝄞💩', -7)",
+    "dataset": null,
+    "bindings": {},
+    "result": "     𝄞💩"
+}
\ No newline at end of file
diff --git a/test/test-suite/groups/function-pad/case009.json b/test/test-suite/groups/function-pad/case009.json
new file mode 100644
index 0000000..9d96e8d
--- /dev/null
+++ b/test/test-suite/groups/function-pad/case009.json
@@ -0,0 +1,6 @@
+{
+    "expr": "$pad('a𝄞💩', 7, '💩')",
+    "dataset": null,
+    "bindings": {},
+    "result": "a𝄞💩💩💩💩💩"
+}
\ No newline at end of file
diff --git a/test/test-suite/groups/function-pad/case010.json b/test/test-suite/groups/function-pad/case010.json
new file mode 100644
index 0000000..e886378
--- /dev/null
+++ b/test/test-suite/groups/function-pad/case010.json
@@ -0,0 +1,6 @@
+{
+    "expr": "$pad('a𝄞💩', 6, '💩')",
+    "dataset": null,
+    "bindings": {},
+    "result": "a𝄞💩💩💩💩"
+}
\ No newline at end of file
diff --git a/test/test-suite/groups/function-split/case018.json b/test/test-suite/groups/function-split/case018.json
new file mode 100644
index 0000000..a28ab15
--- /dev/null
+++ b/test/test-suite/groups/function-split/case018.json
@@ -0,0 +1,10 @@
+{
+    "expr": "$split('12💩345💩6', '💩')",
+    "dataset": null,
+    "bindings": {},
+    "result": [
+        "12",
+        "345",
+        "6"
+    ]
+}
\ No newline at end of file
diff --git a/test/test-suite/groups/function-substring/case004.json b/test/test-suite/groups/function-substring/case004.json
new file mode 100644
index 0000000..074ac23
--- /dev/null
+++ b/test/test-suite/groups/function-substring/case004.json
@@ -0,0 +1,6 @@
+{
+    "expr": "$substring('𝄞💩', 0, 2)",
+    "dataset": null,
+    "bindings": {},
+    "result": "𝄞💩"
+}
\ No newline at end of file
diff --git a/test/test-suite/groups/function-substring/case005.json b/test/test-suite/groups/function-substring/case005.json
new file mode 100644
index 0000000..8f5a5c0
--- /dev/null
+++ b/test/test-suite/groups/function-substring/case005.json
@@ -0,0 +1,6 @@
+{
+    "expr": "$substring('𝄞💩', 0, 1)",
+    "dataset": null,
+    "bindings": {},
+    "result": "𝄞"
+}
\ No newline at end of file
diff --git a/test/test-suite/groups/function-substring/case006.json b/test/test-suite/groups/function-substring/case006.json
new file mode 100644
index 0000000..cce4030
--- /dev/null
+++ b/test/test-suite/groups/function-substring/case006.json
@@ -0,0 +1,6 @@
+{
+    "expr": "$substring('𝄞💩', 1, 1)",
+    "dataset": null,
+    "bindings": {},
+    "result": "💩"
+}
\ No newline at end of file
diff --git a/test/test-suite/groups/function-substring/case007.json b/test/test-suite/groups/function-substring/case007.json
new file mode 100644
index 0000000..d11a371
--- /dev/null
+++ b/test/test-suite/groups/function-substring/case007.json
@@ -0,0 +1,6 @@
+{
+    "expr": "$substring(\"hello world\", -5)",
+    "dataset": "dataset5",
+    "bindings": {},
+    "result": "world"
+}
\ No newline at end of file