Repository: FontoXML/fontoxpath (ts)

Issue:
Problem with `tokenize` or regexs
I ran into this issue when the [`functx:lines` function](http://www.xqueryfunctions.com/xq/functx_lines.html) didn't seem to work for me, giving me an extra blank line after each line.

But I can find a simple reproducer using an example from: https://www.altova.com/xpath-xquery-reference/fn-tokenize which says:

>  For example:
> `fn:tokenize("abracadabra", "(ab)|(a)")` returns `("", "r", "c", "d", "r", "")`

But with fontoxpath:

```js
var fontoxpath = require("fontoxpath")

console.log(fontoxpath.evaluateXPathToStrings(
'fn:tokenize("abracadabra", "(ab)|(a)")',
 null, 
undefined, 
undefined,
 {language: fontoxpath.evaluateXPath.XQUERY_3_1_LANGUAGE}))
```

Output:

```js
[
  '',          'ab',
  'undefined', 'r',
  'undefined', 'a',
  'c',         'undefined',
  'a',         'd',
  'ab',        'undefined',
  'r',         'undefined',
  'a',         ''
]
```

Looks like the captures are being incorrectly returned as part of the results of `tokenize`.

Pull request:
Fix tokenize with capture groups
Fixes a few tests as well!

Also fixes #635. Thanks @rrthomas for finding it!

Patch:
diff --git a/src/expressions/functions/builtInFunctions_string.ts b/src/expressions/functions/builtInFunctions_string.ts
index f347b16d..291c775b 100644
--- a/src/expressions/functions/builtInFunctions_string.ts
+++ b/src/expressions/functions/builtInFunctions_string.ts
@@ -355,11 +355,25 @@ const fnTokenize: FunctionDefinitionType = (
 	const patternString: string = pattern.first().value;
 
 	const regex = compileJSRegex(patternString);
+	regex.lastIndex = 0;
+	const results = [];
+	let result = regex.exec(inputString);
+
+	let index = 0;
+
+	// Repeatedly execute the regex to get all the outputs
+	// TODO: with this approach it should not be too difficult to actually use XSPattern here.
+	while (result) {
+		results.push(inputString.slice(index, result.index));
+		index = regex.lastIndex;
+		result = regex.exec(inputString);
+	}
+
+	// Don't forget the last split part!
+	results.push(inputString.slice(index));
 
 	return sequenceFactory.create(
-		inputString
-			.split(regex)
-			.map((token: string) => createAtomicValue(token, ValueType.XSSTRING)),
+		results.map((token: string) => createAtomicValue(token, ValueType.XSSTRING)),
 	);
 };
 


Test patch:
diff --git a/test/assets/unrunnableTestCases.csv b/test/assets/unrunnableTestCases.csv
index da65a04c..6e984c55 100644
--- a/test/assets/unrunnableTestCases.csv
+++ b/test/assets/unrunnableTestCases.csv
@@ -1625,7 +1625,6 @@ cbcl-timezone-from-date-007,Error: XPST0017: Function Q{http://www.w3.org/2005/x
 fn-tokenize-2,AssertionError: expected [Function] to throw error matching /FORX0001/ but got 'Not implemented: Using flags in token…'
 fn-tokenize-5,Error: Not implemented: Using flags in tokenize is not supported
 fn-tokenize-6,Error: Not implemented: Using flags in tokenize is not supported
-fn-tokenize-9,AssertionError: string-join(fn:tokenize("abracadabra", "(ab)|(a)"), '#'): expected '#ab##r##a#c##a#d#ab##r##a#' to equal '#r#c#d#r#'
 fn-tokenize-31,Error: Not implemented: Using flags in tokenize is not supported
 fn-tokenize-32,Error: Not implemented: Using flags in tokenize is not supported
 fn-tokenize-35,Error: Not implemented: Using flags in tokenize is not supported
@@ -3839,8 +3838,6 @@ functx-functx-dynamic-path-all,Error: No selector counterpart for: computedDocum
 functx-functx-get-matches-1,AssertionError: declare namespace functx = "http://www.example.com/";                     (:~ : Splits a string into matching and non-matching regions : : @author Priscilla Walmsley, Datypic : @version 1.0 : @see http://www.xqueryfunctions.com/xq/functx_get-matches-and-non-matches.html : @param $string the string to split : @param $regex the pattern :)          declare function functx:get-matches-and-non-matches ( $string as xs:string? , $regex as xs:string ) as element()* { let $iomf := functx:index-of-match-first($string, $regex) return if (empty($iomf)) then <non-match>{$string}</non-match> else if ($iomf > 1) then (<non-match>{substring($string,1,$iomf - 1)}</non-match>, functx:get-matches-and-non-matches( substring($string,$iomf),$regex)) else let $length := string-length($string) - string-length(functx:replace-first($string, $regex,'')) return (<match>{substring($string,1,$length)}</match>, if (string-length($string) > $length) then functx:get-matches-and-non-matches( substring($string,$length + 1),$regex) else ()) } ;                     (:~ : Return the matching regions of a string : : @author Priscilla Walmsley, Datypic : @version 1.0 : @see http://www.xqueryfunctions.com/xq/functx_get-matches.html : @param $string the string to split : @param $regex the pattern :)          declare function functx:get-matches ( $string as xs:string? , $regex as xs:string ) as xs:string* { functx:get-matches-and-non-matches($string,$regex)/ string(self::match) } ;                     (:~ : The first position of a matching substring : : @author Priscilla Walmsley, Datypic : @version 1.0 : @see http://www.xqueryfunctions.com/xq/functx_index-of-match-first.html : @param $arg the string : @param $pattern the pattern to match :)          declare function functx:index-of-match-first ( $arg as xs:string? , $pattern as xs:string ) as xs:integer? { if (matches($arg,$pattern)) then string-length(tokenize($arg, $pattern)[1]) + 1 else () } ;                     (:~ : Replaces the first match of a pattern : : @author Priscilla Walmsley, Datypic : @version 1.0 : @see http://www.xqueryfunctions.com/xq/functx_replace-first.html : @param $arg the entire string to change : @param $pattern the pattern of characters to replace : @param $replacement the replacement string :)          declare function functx:replace-first ( $arg as xs:string? , $pattern as xs:string , $replacement as xs:string ) as xs:string { replace($arg, concat('(^.*?)', $pattern), concat('$1',$replacement)) } ;                     (functx:get-matches( 'abc123def', '\d+')): expected '123  ' to equal ' 123 '
 functx-functx-get-matches-2,AssertionError: declare namespace functx = "http://www.example.com/";                     (:~ : Splits a string into matching and non-matching regions : : @author Priscilla Walmsley, Datypic : @version 1.0 : @see http://www.xqueryfunctions.com/xq/functx_get-matches-and-non-matches.html : @param $string the string to split : @param $regex the pattern :)          declare function functx:get-matches-and-non-matches ( $string as xs:string? , $regex as xs:string ) as element()* { let $iomf := functx:index-of-match-first($string, $regex) return if (empty($iomf)) then <non-match>{$string}</non-match> else if ($iomf > 1) then (<non-match>{substring($string,1,$iomf - 1)}</non-match>, functx:get-matches-and-non-matches( substring($string,$iomf),$regex)) else let $length := string-length($string) - string-length(functx:replace-first($string, $regex,'')) return (<match>{substring($string,1,$length)}</match>, if (string-length($string) > $length) then functx:get-matches-and-non-matches( substring($string,$length + 1),$regex) else ()) } ;                     (:~ : Return the matching regions of a string : : @author Priscilla Walmsley, Datypic : @version 1.0 : @see http://www.xqueryfunctions.com/xq/functx_get-matches.html : @param $string the string to split : @param $regex the pattern :)          declare function functx:get-matches ( $string as xs:string? , $regex as xs:string ) as xs:string* { functx:get-matches-and-non-matches($string,$regex)/ string(self::match) } ;                     (:~ : The first position of a matching substring : : @author Priscilla Walmsley, Datypic : @version 1.0 : @see http://www.xqueryfunctions.com/xq/functx_index-of-match-first.html : @param $arg the string : @param $pattern the pattern to match :)          declare function functx:index-of-match-first ( $arg as xs:string? , $pattern as xs:string ) as xs:integer? { if (matches($arg,$pattern)) then string-length(tokenize($arg, $pattern)[1]) + 1 else () } ;                     (:~ : Replaces the first match of a pattern : : @author Priscilla Walmsley, Datypic : @version 1.0 : @see http://www.xqueryfunctions.com/xq/functx_replace-first.html : @param $arg the entire string to change : @param $pattern the pattern of characters to replace : @param $replacement the replacement string :)          declare function functx:replace-first ( $arg as xs:string? , $pattern as xs:string , $replacement as xs:string ) as xs:string { replace($arg, concat('(^.*?)', $pattern), concat('$1',$replacement)) } ;                     (functx:get-matches( 'abc123def', '\d')): expected ' 1 3  2' to equal ' 1 2 3 '
 functx-functx-get-matches-all,AssertionError: declare namespace functx = "http://www.example.com/";                     (:~ : Splits a string into matching and non-matching regions : : @author Priscilla Walmsley, Datypic : @version 1.0 : @see http://www.xqueryfunctions.com/xq/functx_get-matches-and-non-matches.html : @param $string the string to split : @param $regex the pattern :)          declare function functx:get-matches-and-non-matches ( $string as xs:string? , $regex as xs:string ) as element()* { let $iomf := functx:index-of-match-first($string, $regex) return if (empty($iomf)) then <non-match>{$string}</non-match> else if ($iomf > 1) then (<non-match>{substring($string,1,$iomf - 1)}</non-match>, functx:get-matches-and-non-matches( substring($string,$iomf),$regex)) else let $length := string-length($string) - string-length(functx:replace-first($string, $regex,'')) return (<match>{substring($string,1,$length)}</match>, if (string-length($string) > $length) then functx:get-matches-and-non-matches( substring($string,$length + 1),$regex) else ()) } ;                     (:~ : Return the matching regions of a string : : @author Priscilla Walmsley, Datypic : @version 1.0 : @see http://www.xqueryfunctions.com/xq/functx_get-matches.html : @param $string the string to split : @param $regex the pattern :)          declare function functx:get-matches ( $string as xs:string? , $regex as xs:string ) as xs:string* { functx:get-matches-and-non-matches($string,$regex)/ string(self::match) } ;                     (:~ : The first position of a matching substring : : @author Priscilla Walmsley, Datypic : @version 1.0 : @see http://www.xqueryfunctions.com/xq/functx_index-of-match-first.html : @param $arg the string : @param $pattern the pattern to match :)          declare function functx:index-of-match-first ( $arg as xs:string? , $pattern as xs:string ) as xs:integer? { if (matches($arg,$pattern)) then string-length(tokenize($arg, $pattern)[1]) + 1 else () } ;                     (:~ : Replaces the first match of a pattern : : @author Priscilla Walmsley, Datypic : @version 1.0 : @see http://www.xqueryfunctions.com/xq/functx_replace-first.html : @param $arg the entire string to change : @param $pattern the pattern of characters to replace : @param $replacement the replacement string :)          declare function functx:replace-first ( $arg as xs:string? , $pattern as xs:string , $replacement as xs:string ) as xs:string { replace($arg, concat('(^.*?)', $pattern), concat('$1',$replacement)) } ;                     (functx:get-matches( 'abc123def', '\d+'), functx:get-matches( 'abc123def', '\d'), functx:get-matches( 'abc123def', '[a-z]{2}')): expected '123    1 3  2 ab  de ' to equal ' 123   1 2 3  ab  de '
-functx-functx-line-count-2,AssertionError: Expected XPath declare namespace functx = "http://www.example.com/";                     (:~ : The number of lines : : @author Priscilla Walmsley, Datypic : @version 1.0 : @see http://www.xqueryfunctions.com/xq/functx_line-count.html : @param $arg the string to test :)          declare function functx:line-count ( $arg as xs:string? ) as xs:integer { count(functx:lines($arg)) } ;                     (:~ : Split a string into separate lines : : @author Priscilla Walmsley, Datypic : @version 1.0 : @see http://www.xqueryfunctions.com/xq/functx_lines.html : @param $arg the string to split :)          declare function functx:lines ( $arg as xs:string? ) as xs:string* { tokenize($arg, '(\r\n?|\n\r?)') } ;          let $lines := 'a              value              on              many              lines' return (functx:line-count($lines))        to resolve to 5: expected false to be true
-functx-functx-line-count-all,AssertionError: declare namespace functx = "http://www.example.com/";                     (:~ : The number of lines : : @author Priscilla Walmsley, Datypic : @version 1.0 : @see http://www.xqueryfunctions.com/xq/functx_line-count.html : @param $arg the string to test :)          declare function functx:line-count ( $arg as xs:string? ) as xs:integer { count(functx:lines($arg)) } ;                     (:~ : Split a string into separate lines : : @author Priscilla Walmsley, Datypic : @version 1.0 : @see http://www.xqueryfunctions.com/xq/functx_lines.html : @param $arg the string to split :)          declare function functx:lines ( $arg as xs:string? ) as xs:string* { tokenize($arg, '(\r\n?|\n\r?)') } ;          let $lines := 'a                      value                      on                      many                      lines'           return (functx:line-count('a value'), functx:line-count($lines)): expected '1 9' to equal '1 5'
 functx-functx-next-day-1,Error: Not implemented: adding durations to xs:date
 functx-functx-next-day-2,Error: Not implemented: adding durations to xs:date
 functx-functx-next-day-all,Error: Not implemented: adding durations to xs:date
diff --git a/test/specs/parsing/functions/functions.string.tests.ts b/test/specs/parsing/functions/functions.string.tests.ts
index da3f5f9d..ac20624b 100644
--- a/test/specs/parsing/functions/functions.string.tests.ts
+++ b/test/specs/parsing/functions/functions.string.tests.ts
@@ -95,6 +95,13 @@ describe('functions over strings', () => {
 			));
 	});
 
+	it('tokenize("abracadabra", "(ab)|(a)") returns ("","r","c","d","r","")', () => {
+		chai.assert.sameOrderedMembers(
+			evaluateXPathToStrings('fn:tokenize("abracadabra", "(ab)|(a)")'),
+			['', 'r', 'c', 'd', 'r', ''],
+		);
+	});
+
 	describe('normalize-space()', () => {
 		it('Returns the value of $arg with whitespace normalized by stripping leading and trailing whitespace and replacing sequences of one or more than one whitespace character with a single space, #x20.', () =>
 			chai.assert.equal(