Repository: jsonata-js/jsonata (js)

Issue:
Defining functions for use with $match etc.
As discussed in [this thread](https://github.com/jsonata-js/jsonata/issues/177), the functions `$match`, `$contains`, `$split` and `$replace` all take a function that returns the matches of a particular pattern in a string. Regular expressions are one type of matching function but users can provide their own. I tried this out (to check that jsonata-go is doing the right thing) and found it quite tricky to get working.

I expected this expression to return one match:

```
(
    $f := function($arg) {
        {
            "match": "ab",
            "start": 0,
            "end": 2,
            "groups": [],
            "next": function() { nothing }
        }
    };

    $match("abracadabra", $f)
)
```

But instead it returned an error: `t is not a function`. So I re-ran the expression in the JavaScript console and got the slightly longer error: `Uncaught TypeError: regex is not a function` (`regex` is the name of the argument to `functionMatch`).

Ok, so lambdas can't be used as matching functions. Not a problem. I rewrote the function in JavaScript, added it to JSONata with `registerFunction`, and used that as the second argument to `$match`. Same error!

Looking at the code, I saw that `functionMatch` calls its `regex` argument directly. This means that `regex` has to be a native JavaScript function. But `registerFunction` wraps the function passed to it in an object. I tried using `assign` instead of `registerFunction` and that fixed the problem.

I have one observation and one question about this:

1. Functions added via `registerFunction` should work. It doesn't make sense for the same matching function to be valid when bound with `assign` and invalid when bound with `registerFunction`.

2. Is there a reason why lambdas can't be used as matching functions or is it just an oversight?

p.s. I only tried `$match` and `$contains` but I can see from the code that `$split` and `$replace` will have the same issues. They all call the matching function directly.


Pull request description:
add support for custom matcher functions
The four functions `$match`, `$contains`, `$split` and `$replace` can take a regex argument.  The regex syntax `/.../` is in fact just some syntactic sugar for a higher-order function that generates a 'matcher' function.  This matcher function, when invoked on a string, will return a well defined structure (which I haven't documented yet :blush:) which describes the match(es).  It was designed so that these four functions could work with any pluggable matching scheme, not just the built-in regex, provided the matcher returned the correct object structure.  That's the theory, but I never got round to testing it.  This PR fixes that.  Custom matchers can now be implemented in jsonata as lambdas or as external javascript functions.

Resolves #213 

Patch:
diff --git a/jsonata.js b/jsonata.js
index 15824bc..7443ac0 100644
--- a/jsonata.js
+++ b/jsonata.js
@@ -2390,6 +2390,25 @@ var jsonata = (function() {
         return closure;
     }
 
+    /**
+     * Evaluate the matcher function against the str arg
+     *
+     * @param {*} matcher - matching function (native or lambda)
+     * @param {string} str - the string to match against
+     * @returns {object} - structure that represents the match(es)
+     */
+    function* evaluateMatcher(matcher, str) {
+        var result = yield * apply(matcher, [str]);
+        if(result && !(typeof result.start === 'number' || result.end === 'number' || Array.isArray(result.groups) || isFunction(result.next))) {
+            // the matcher function didn't return the correct structure
+            throw {
+                code: "T1010",
+                stack: (new Error()).stack,
+            };
+        }
+        return result;
+    }
+
     /**
      * Evaluate variable against input data
      * @param {Object} expr - JSONata expression
@@ -3243,7 +3262,7 @@ var jsonata = (function() {
      * @param {String} token - substring or regex to find
      * @returns {Boolean} - true if str contains token
      */
-    function functionContains(str, token) {
+    function* functionContains(str, token) {
         // undefined inputs always return undefined
         if(typeof str === 'undefined') {
             return undefined;
@@ -3254,7 +3273,7 @@ var jsonata = (function() {
         if(typeof token === 'string') {
             result = (str.indexOf(token) !== -1);
         } else {
-            var matches = token(str);
+            var matches = yield * evaluateMatcher(token, str);
             result = (typeof matches !== 'undefined');
         }
 
@@ -3268,7 +3287,7 @@ var jsonata = (function() {
      * @param {Integer} [limit] - max number of matches to return
      * @returns {Array} The array of match objects
      */
-    function functionMatch(str, regex, limit) {
+    function* functionMatch(str, regex, limit) {
         // undefined inputs always return undefined
         if(typeof str === 'undefined') {
             return undefined;
@@ -3288,7 +3307,7 @@ var jsonata = (function() {
 
         if(typeof limit === 'undefined' || limit > 0) {
             var count = 0;
-            var matches = regex(str);
+            var matches = yield * evaluateMatcher(regex, str);
             if (typeof matches !== 'undefined') {
                 while (typeof matches !== 'undefined' && (typeof limit === 'undefined' || count < limit)) {
                     result.push({
@@ -3296,7 +3315,7 @@ var jsonata = (function() {
                         index: matches.start,
                         groups: matches.groups
                     });
-                    matches = matches.next();
+                    matches = yield * evaluateMatcher(matches.next);
                     count++;
                 }
             }
@@ -3409,7 +3428,7 @@ var jsonata = (function() {
                 }
                 result += str.substring(position);
             } else {
-                var matches = pattern(str);
+                var matches = yield * evaluateMatcher(pattern, str);
                 if (typeof matches !== 'undefined') {
                     while (typeof matches !== 'undefined' && (typeof limit === 'undefined' || count < limit)) {
                         result += str.substring(position, matches.start);
@@ -3427,7 +3446,7 @@ var jsonata = (function() {
                         }
                         position = matches.start + matches.match.length;
                         count++;
-                        matches = matches.next();
+                        matches = yield * evaluateMatcher(matches.next);
                     }
                     result += str.substring(position);
                 } else {
@@ -3493,7 +3512,7 @@ var jsonata = (function() {
      * @param {Integer} [limit] - max number of substrings
      * @returns {Array} The array of string
      */
-    function functionSplit(str, separator, limit) {
+    function* functionSplit(str, separator, limit) {
         // undefined inputs always return undefined
         if(typeof str === 'undefined') {
             return undefined;
@@ -3516,13 +3535,13 @@ var jsonata = (function() {
                 result = str.split(separator, limit);
             } else {
                 var count = 0;
-                var matches = separator(str);
+                var matches = yield * evaluateMatcher(separator, str);
                 if (typeof matches !== 'undefined') {
                     var start = 0;
                     while (typeof matches !== 'undefined' && (typeof limit === 'undefined' || count < limit)) {
                         result.push(str.substring(start, matches.start));
                         start = matches.end;
-                        matches = matches.next();
+                        matches = yield * evaluateMatcher(matches.next);
                         count++;
                     }
                     if(typeof limit === 'undefined' || count < limit) {
@@ -4828,6 +4847,7 @@ var jsonata = (function() {
         "T1007": "Attempted to partially apply a non-function. Did you mean ${{{token}}}?",
         "T1008": "Attempted to partially apply a non-function",
         "D1009": "Multiple key definitions evaluate to same key: {{value}}",
+        "T1010": "The matcher function argument passed to function {{token}} does not return the correct object structure",
         "T2001": "The left side of the {{token}} operator must evaluate to a number",
         "T2002": "The right side of the {{token}} operator must evaluate to a number",
         "T2003": "The left side of the range operator (..) must evaluate to an integer",


Test patch:
diff --git a/test/implementation-tests.js b/test/implementation-tests.js
index 171fb13..efe6de5 100644
--- a/test/implementation-tests.js
+++ b/test/implementation-tests.js
@@ -300,6 +300,7 @@ describe("Tests that bind Javascript functions", () => {
             expect(result).to.deep.equal(expected);
         });
     });
+
     describe("Partially apply user-defined Javascript function", function() {
         it("should return result object", function() {
             var expr = jsonata(
@@ -317,6 +318,69 @@ describe("Tests that bind Javascript functions", () => {
             expect(result).to.deep.equal(expected);
         });
     });
+
+    describe("User defined matchers", function() {
+        var repeatingLetters = function(char, repeat) {
+            // custom matcher to match `repeat` contiguous occurrences of `char`
+            var chars = char.repeat(repeat);
+            var match = function(str, offset) {
+                var pos = str.indexOf(chars, (offset || 0));
+                if (pos === -1) {
+                    return;
+                } else {
+                    return {
+                        match: chars,
+                        start: pos,
+                        end: pos + chars.length,
+                        groups: [],
+                        next: function () {
+                            return match(str, pos + chars.length);
+                        }
+                    };
+                }
+            };
+            return match;
+        };
+
+        it("should match using a custom matcher", function() {
+            var expr = jsonata("$match('LLANFAIRPWLLGWYNGYLLGOGERYCHWYRNDROBWLLLLANTYSILIOGOGOGOCH', $repeatingLetters('L', 2))");
+            expr.registerFunction("repeatingLetters", repeatingLetters);
+            var result = expr.evaluate();
+            var expected = [
+                {"match": "LL", "index": 0, "groups": []},
+                {"match": "LL", "index": 10, "groups": []},
+                {"match": "LL", "index": 18, "groups": []},
+                {"match": "LL", "index": 37, "groups": []},
+                {"match": "LL", "index": 39, "groups": []}
+            ];
+            expect(result).to.deep.equal(expected);
+        });
+
+        it("should split using a custom matcher", function() {
+            var expr = jsonata("$split('LLANFAIRPWLLGWYNGYLLGOGERYCHWYRNDROBWLLLLANTYSILIOGOGOGOCH', $repeatingLetters('L', 2))");
+            expr.registerFunction("repeatingLetters", repeatingLetters);
+            var result = expr.evaluate();
+            var expected = ["","ANFAIRPW","GWYNGY","GOGERYCHWYRNDROBW","","ANTYSILIOGOGOGOCH"];
+            expect(result).to.deep.equal(expected);
+        });
+
+        it("should replace using a custom matcher", function() {
+            var expr = jsonata("$replace('LLANFAIRPWLLGWYNGYLLGOGERYCHWYRNDROBWLLLLANTYSILIOGOGOGOCH', $repeatingLetters('L', 2), 'Ỻ')");
+            expr.registerFunction("repeatingLetters", repeatingLetters);
+            var result = expr.evaluate();
+            var expected = "ỺANFAIRPWỺGWYNGYỺGOGERYCHWYRNDROBWỺỺANTYSILIOGOGOGOCH";
+            expect(result).to.deep.equal(expected);
+        });
+
+        it("should test inclusion using a custom matcher", function() {
+            var expr = jsonata("$contains('LLANFAIRPWLLGWYNGYLLGOGERYCHWYRNDROBWLLLLANTYSILIOGOGOGOCH', $repeatingLetters('L', 4))");
+            expr.registerFunction("repeatingLetters", repeatingLetters);
+            var result = expr.evaluate();
+            var expected = true;
+            expect(result).to.deep.equal(expected);
+        });
+
+    });
 });
 
 describe("Tests that are specific to a Javascript runtime", () => {
diff --git a/test/run-test-suite-async.js b/test/run-test-suite-async.js
index f03a706..1d4122a 100644
--- a/test/run-test-suite-async.js
+++ b/test/run-test-suite-async.js
@@ -58,7 +58,7 @@ function jsonataPromise(expr, data, bindings) {
 describe("JSONata Test Suite - async mode", () => {
     // Iterate over all groups of tests
     groups.forEach(group => {
-        let casenames = fs.readdirSync(path.join(__dirname, "test-suite", "groups", group));
+        let casenames = fs.readdirSync(path.join(__dirname, "test-suite", "groups", group)).filter((name) => name.endsWith(".json"));
         // Read JSON file containing all cases for this group
         let cases = casenames.map((name) => readJSON(path.join("test-suite", "groups", group), name));
         describe("Group: " + group, () => {
@@ -67,6 +67,11 @@ describe("JSONata Test Suite - async mode", () => {
                 // Extract the current test case of interest
                 let testcase = cases[i];
 
+                // if the testcase references an external jsonata file, read it in
+                if(testcase['expr-file']) {
+                    testcase.expr = fs.readFileSync(path.join(__dirname, "test-suite", "groups", group, testcase['expr-file'])).toString();
+                }
+
                 // Create a test based on the data in this testcase
                 it(casenames[i]+": "+testcase.expr, function() {
                     var expr;
diff --git a/test/run-test-suite.js b/test/run-test-suite.js
index ddf6ff5..2fc973a 100644
--- a/test/run-test-suite.js
+++ b/test/run-test-suite.js
@@ -40,7 +40,7 @@ datasetnames.forEach((name) => {
 describe("JSONata Test Suite", () => {
     // Iterate over all groups of tests
     groups.forEach(group => {
-        let casenames = fs.readdirSync(path.join(__dirname, "test-suite", "groups", group));
+        let casenames = fs.readdirSync(path.join(__dirname, "test-suite", "groups", group)).filter((name) => name.endsWith(".json"));
         // Read JSON file containing all cases for this group
         let cases = casenames.map((name) => readJSON(path.join("test-suite", "groups", group), name));
         describe("Group: " + group, () => {
@@ -49,6 +49,11 @@ describe("JSONata Test Suite", () => {
                 // Extract the current test case of interest
                 let testcase = cases[i];
 
+                // if the testcase references an external jsonata file, read it in
+                if(testcase['expr-file']) {
+                    testcase.expr = fs.readFileSync(path.join(__dirname, "test-suite", "groups", group, testcase['expr-file'])).toString();
+                }
+
                 // Create a test based on the data in this testcase
                 it(casenames[i]+": "+testcase.expr, function() {
                     var expr;
diff --git a/test/test-suite/groups/matchers/case000.json b/test/test-suite/groups/matchers/case000.json
new file mode 100644
index 0000000..a025766
--- /dev/null
+++ b/test/test-suite/groups/matchers/case000.json
@@ -0,0 +1,32 @@
+{
+  "expr-file": "case000.jsonata",
+  "dataset": null,
+  "bindings": {},
+  "result": [
+    {
+      "match": "a",
+      "index": 0,
+      "groups": []
+    },
+    {
+      "match": "a",
+      "index": 3,
+      "groups": []
+    },
+    {
+      "match": "a",
+      "index": 5,
+      "groups": []
+    },
+    {
+      "match": "a",
+      "index": 7,
+      "groups": []
+    },
+    {
+      "match": "a",
+      "index": 10,
+      "groups": []
+    }
+  ]
+}
\ No newline at end of file
diff --git a/test/test-suite/groups/matchers/case000.jsonata b/test/test-suite/groups/matchers/case000.jsonata
new file mode 100644
index 0000000..d5c8d70
--- /dev/null
+++ b/test/test-suite/groups/matchers/case000.jsonata
@@ -0,0 +1,18 @@
+(
+    $generateMatcher := function($ch) {
+      $match := function($str, $offset) {(
+        $before := $substringBefore($str, $ch);
+        $start := $length($before) + ($exists($offset) ? $offset : 0);
+        $end := $start + $length($ch);
+        $before != $str and $length($ch) > 0 ? {
+          'match': $ch,
+          'start': $start,
+          'end': $end,
+          'groups': [],
+          'next': function() {$match($substringAfter($str, $ch), $end)}
+        }
+      )}
+    };
+
+    $match("abracadabra", $generateMatcher('a'))
+)
\ No newline at end of file
diff --git a/test/test-suite/groups/matchers/case001.json b/test/test-suite/groups/matchers/case001.json
new file mode 100644
index 0000000..282586a
--- /dev/null
+++ b/test/test-suite/groups/matchers/case001.json
@@ -0,0 +1,6 @@
+{
+  "expr": "$split('some text', $uppercase)",
+  "dataset": null,
+  "bindings": {},
+  "code": "T1010"
+}
\ No newline at end of file