Repository: rayokota/jsonata-python

Issue:
 It can't evaluate a regular expression
Hey! Version 0.5.1 and python 3.12

My code works in native env
https://try.jsonata.org/OYLsxCTGr

But I have troubles when run with jsonata-python. It can't parse a regular expression:

```
Traceback (most recent call last):
  File "/Users/user/code/proj/venv/lib/python3.12/site-packages/jsonata/functions.py", line 2203, in function_eval
    ast = jsonata.Jsonata(expr)
          ^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/code/proj/venv/lib/python3.12/site-packages/jsonata/jsonata.py", line 1888, in __init__
    self.ast = self.parser.parse(expr)  # , optionsRecover);
               ^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/code/projvenv/lib/python3.12/site-packages/jsonata/parser.py", line 1377, in parse
    self.advance()
  File "/Users/user/code/proj/venv/lib/python3.12/site-packages/jsonata/parser.py", line 283, in advance
    next_token = self.lexer.next(infix)
                 ^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/code/proj/venv/lib/python3.12/site-packages/jsonata/tokenizer.py", line 185, in next
    return self.create("regex", self.scan_regex())
                                ^^^^^^^^^^^^^^^^^
  File "/Users/user/code/proj/venv/lib/python3.12/site-packages/jsonata/tokenizer.py", line 141, in scan_regex
    current_char = self.path[self.position]
                   ~~~~~~~~~^^^^^^^^^^^^^^^
IndexError: string index out of range

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/Users/user/code/proj/main.py", line 124, in <module>
    result = expr.evaluate(jsonMap)
             ^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/code/proj/venv/lib/python3.12/site-packages/jsonata/jsonata.py", line 1978, in evaluate
    raise err
  File "/Users/user/code/proj/venv/lib/python3.12/site-packages/jsonata/jsonata.py", line 1969, in evaluate
    it = self.eval(self.ast, input, exec_env)
         ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/code/proj/venv/lib/python3.12/site-packages/jsonata/jsonata.py", line 234, in eval
    return self.get_per_thread_instance()._eval(expr, input, environment)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/code/proj/venv/lib/python3.12/site-packages/jsonata/jsonata.py", line 272, in _eval
    result = self.evaluate_block(expr, input, environment)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/code/proj/venv/lib/python3.12/site-packages/jsonata/jsonata.py", line 1097, in evaluate_block
    result = self.eval(ex, input, frame)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/code/proj/venv/lib/python3.12/site-packages/jsonata/jsonata.py", line 234, in eval
    return self.get_per_thread_instance()._eval(expr, input, environment)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/code/proj/venv/lib/python3.12/site-packages/jsonata/jsonata.py", line 274, in _eval
    result = self.evaluate_bind_expression(expr, input, environment)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/code/proj/venv/lib/python3.12/site-packages/jsonata/jsonata.py", line 1058, in evaluate_bind_expression
    value = self.eval(expr.rhs, input, environment)
            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/code/proj/venv/lib/python3.12/site-packages/jsonata/jsonata.py", line 234, in eval
    return self.get_per_thread_instance()._eval(expr, input, environment)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/code/proj/venv/lib/python3.12/site-packages/jsonata/jsonata.py", line 278, in _eval
    result = self.evaluate_function(expr, input, environment, utils.Utils.NONE)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/code/proj/venv/lib/python3.12/site-packages/jsonata/jsonata.py", line 1391, in evaluate_function
    raise jex
  File "/Users/user/code/proj/venv/lib/python3.12/site-packages/jsonata/jsonata.py", line 1383, in evaluate_function
    result = self.apply(proc, evaluated_args, input, environment)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/code/projvenv/lib/python3.12/site-packages/jsonata/jsonata.py", line 1406, in apply
    result = self.apply_inner(proc, args, input, environment)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/code/proj/venv/lib/python3.12/site-packages/jsonata/jsonata.py", line 1489, in apply_inner
    raise err
  File "/Users/user/code/proj/venv/lib/python3.12/site-packages/jsonata/jsonata.py", line 1471, in apply_inner
    result = proc.call(input, validated_args)
             ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/code/proj/venv/lib/python3.12/site-packages/jsonata/jsonata.py", line 161, in call
    return functions.Functions._call(self.method, self.nargs, args)
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/user/code/proj/venv/lib/python3.12/site-packages/jsonata/functions.py", line 2073, in _call
    res = m(*call_args)
          ^^^^^^^^^^^^^
  File "/Users/user/code/proj/venv/lib/python3.12/site-packages/jsonata/functions.py", line 2207, in function_eval
    raise jexception.JException("D3120", -1)
jsonata.jexception.JException: Syntax error in expression passed to Object eval: None
```

Discussion hints:

Solution patch:
diff --git a/src/jsonata/tokenizer.py b/src/jsonata/tokenizer.py
index bf77a48..7ff37b4 100644
--- a/src/jsonata/tokenizer.py
+++ b/src/jsonata/tokenizer.py
@@ -138,7 +138,10 @@ class Tokenizer:
                 start = self.position
                 while current_char == 'i' or current_char == 'm':
                     self.position += 1
-                    current_char = self.path[self.position]
+                    if self.position < self.length:
+                        current_char = self.path[self.position]
+                    else:
+                        current_char = None
                 flags = self.path[start:self.position] + 'g'
 
                 # Convert flags to Java Pattern flags


Test patch:
diff --git a/tests/string_test.py b/tests/string_test.py
index 9b09b21..7eb4deb 100644
--- a/tests/string_test.py
+++ b/tests/string_test.py
@@ -38,6 +38,10 @@ class TestString:
         assert jsonata.Jsonata("$replace('h.ello', '.', '')").evaluate(None) == "hello"
         assert jsonata.Jsonata("$replace('h.e.l.l.o', '.', '',2)").evaluate(None) == "hel.l.o"
 
+    def test_regex(self):
+        assert (jsonata.Jsonata("($matcher := $eval('/^' & 'foo' & '/i'); $.$spread()[$.$keys() ~> $matcher])")
+                .evaluate({"foo": 1, "bar": 2}) == {"foo": 1})
+
     #
     # Additional $split tests
     #