Repository: MiSawa/xq
Pull request #114 (2022-03-13 01:01:27)

Implement test, match, capture, and scan
Some parts of #89


Patch:
diff --git a/Cargo.lock b/Cargo.lock
index 50d8d33..a6d7c80 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -91,6 +91,12 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
 
+[[package]]
+name = "cc"
+version = "1.0.73"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
+
 [[package]]
 name = "cfg-if"
 version = "1.0.0"
@@ -535,6 +541,28 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "onig"
+version = "6.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "67ddfe2c93bb389eea6e6d713306880c7f6dcc99a75b659ce145d962c861b225"
+dependencies = [
+ "bitflags",
+ "lazy_static",
+ "libc",
+ "onig_sys",
+]
+
+[[package]]
+name = "onig_sys"
+version = "69.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5dd3eee045c84695b53b20255bb7317063df090b68e18bfac0abb6c39cf7f33e"
+dependencies = [
+ "cc",
+ "pkg-config",
+]
+
 [[package]]
 name = "ordered-float"
 version = "2.10.0"
@@ -657,6 +685,12 @@ version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "db8bcd96cb740d03149cbad5518db9fd87126a10ab519c011893b1754134c468"
 
+[[package]]
+name = "pkg-config"
+version = "0.3.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "58893f751c9b0412871a09abd62ecd2a00298c6c83befa223ef98c52aef40cbe"
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.16"
@@ -1204,6 +1238,7 @@ dependencies = [
  "num",
  "num-derive",
  "num-traits",
+ "onig",
  "ordered-float",
  "phf",
  "serde",
diff --git a/Cargo.toml b/Cargo.toml
index cdd7f72..c2836a5 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -44,6 +44,7 @@ lexgen_util = "0.10.0"
 time = { version = "0.3.7", features = ["local-offset"] }
 time-fmt = "0.3.4"
 time-tz = { version = "0.5.2", features = ["system", "posix-tz"] }
+onig = { version = "6.3.1", default-features = false }
 
 clap = { version = "3.1.6", features = ["derive"], optional = true }
 clap-verbosity-flag = { version = "1.0.0", optional = true }
diff --git a/prelude.jq b/prelude.jq
index 51c1424..e531faf 100644
--- a/prelude.jq
+++ b/prelude.jq
@@ -91,3 +91,12 @@ def todateiso8601: strftime("%Y-%m-%dT%H:%M:%SZ");
 def fromdate: fromdateiso8601;
 def todate: todateiso8601;
 
+def match($r; $f): __match_impl($r; $f)[];
+def match($r): if $r | type == "array" then match($r[0]; $r[1]) else match($r; null) end;
+def test($r; $f): isempty(match($r; $f)) | not;
+def test($r): if $r | type == "array" then test($r[0]; $r[1]) else test($r; null) end;
+def capture($r; $f): match($r; $f) | [.captures[] | select(.name) | {key: .name, value: .string}] | from_entries;
+def capture($r): if $r | type == "array" then capture($r[0]; $r[1]) else capture($r; null) end;
+def scan($r; $f): match($r; $f + "g") | if .captures | length == 0 then .string else .captures | [.[].string] end;
+def scan($r): if $r | type == "array" then scan($r[0]; $r[1]) else scan($r; null) end;
+
diff --git a/src/intrinsic/mod.rs b/src/intrinsic/mod.rs
index b774b98..1c3a94a 100644
--- a/src/intrinsic/mod.rs
+++ b/src/intrinsic/mod.rs
@@ -28,6 +28,7 @@ mod index;
 #[macro_use]
 mod math;
 mod path;
+mod regex;
 mod string;
 mod time;
 mod unary;
@@ -94,6 +95,7 @@ static INTRINSICS1: phf::Map<&'static str, NamedFn1> = phf_map! {
 };
 static INTRINSICS2: phf::Map<&'static str, NamedFn2> = phf_map! {
     "setpath" => NamedFn2 { name: "setpath", func: path::set_path },
+    "__match_impl" => NamedFn2 { name: "__match_impl", func: regex::match_impl },
 };
 
 pub(crate) fn lookup_intrinsic_fn(
diff --git a/src/intrinsic/regex.rs b/src/intrinsic/regex.rs
new file mode 100644
index 0000000..8eff31e
--- /dev/null
+++ b/src/intrinsic/regex.rs
@@ -0,0 +1,121 @@
+use std::rc::Rc;
+
+use onig::{Regex, RegexOptions, Syntax};
+
+use crate::{
+    vm::{QueryExecutionError, Result},
+    Array, Object, Value,
+};
+
+fn to_string(func: &'static str, v: Value) -> Result<Rc<String>> {
+    match v {
+        Value::String(s) => Ok(s),
+        _ => Err(QueryExecutionError::InvalidArgType(func, v)),
+    }
+}
+
+fn compile_regex(pattern: &str, flags: &str) -> Result<(Regex, bool)> {
+    let mut opt = RegexOptions::REGEX_OPTION_CAPTURE_GROUP;
+    let mut global = false;
+    for c in flags.chars() {
+        match c {
+            'g' => global = true,
+            'i' => opt |= RegexOptions::REGEX_OPTION_IGNORECASE,
+            'm' => opt |= RegexOptions::REGEX_OPTION_MULTILINE,
+            'n' => opt |= RegexOptions::REGEX_OPTION_FIND_NOT_EMPTY,
+            'p' => {
+                opt |= RegexOptions::REGEX_OPTION_SINGLELINE | RegexOptions::REGEX_OPTION_MULTILINE
+            }
+            's' => opt |= RegexOptions::REGEX_OPTION_SINGLELINE,
+            'l' => opt |= RegexOptions::REGEX_OPTION_FIND_LONGEST,
+            'x' => opt |= RegexOptions::REGEX_OPTION_EXTEND,
+            _ => return Err(QueryExecutionError::InvalidRegexFlag(c)),
+        }
+    }
+    Ok((
+        Regex::with_options(pattern, opt, Syntax::default())?,
+        global,
+    ))
+}
+
+pub(crate) fn match_impl(context: Value, pattern: Value, flags: Value) -> Result<Value> {
+    let s = to_string("match", context)?;
+    let pattern = to_string("match", pattern)?;
+    let flags = if flags == Value::Null {
+        Rc::new("".into())
+    } else {
+        to_string("match", flags)?
+    };
+    let (regex, global) = compile_regex(&pattern, &flags)?;
+
+    let mut b2c: Vec<usize> = std::iter::repeat(0).take(s.len() + 1).collect();
+    for i in 0..s.len() {
+        if s.is_char_boundary(i) {
+            b2c[i + 1] = b2c[i] + 1;
+        } else {
+            b2c[i + 1] = b2c[i]
+        }
+    }
+    let mut capture_names = vec![Value::Null; regex.captures_len() + 1]; // +1 for the whole match
+    regex.foreach_name(|name, groups| {
+        let name = Rc::new(name.to_owned());
+        for group in groups {
+            capture_names[*group as usize] = Value::String(name.clone());
+        }
+        true
+    });
+    let arr: Array = regex
+        .captures_iter(&s)
+        .take(if global { usize::MAX } else { 1 })
+        .map(|c| {
+            let (start, end) = c.pos(0).unwrap();
+            let captures: Array = c
+                .iter_pos()
+                .enumerate()
+                .skip(1)
+                .map(|(i, g)| {
+                    let entries: Object = if let Some((start, end)) = g {
+                        [
+                            (Rc::new("offset".into()), Value::number(b2c[start])),
+                            (
+                                Rc::new("length".into()),
+                                Value::number(b2c[end] - b2c[start]),
+                            ),
+                            (
+                                Rc::new("string".into()),
+                                Value::string(s[start..end].into()),
+                            ),
+                            (Rc::new("name".into()), capture_names[i].clone()),
+                        ]
+                    } else {
+                        [
+                            (Rc::new("offset".into()), Value::number(-1)),
+                            (Rc::new("length".into()), Value::number(0)),
+                            (Rc::new("string".into()), Value::Null),
+                            (Rc::new("name".into()), capture_names[i].clone()),
+                        ]
+                    }
+                    .into_iter()
+                    .collect();
+                    entries.into()
+                })
+                .collect();
+            let entries: Object = [
+                (Rc::new("offset".into()), Value::number(b2c[start])),
+                (
+                    Rc::new("length".into()),
+                    Value::number(b2c[end] - b2c[start]),
+                ),
+                (
+                    Rc::new("string".into()),
+                    Value::string(s[start..end].into()),
+                ),
+                (Rc::new("captures".into()), captures.into()),
+            ]
+            .into_iter()
+            .collect();
+            entries.into()
+        })
+        .collect();
+    Ok(arr.into())
+}
diff --git a/src/vm/error.rs b/src/vm/error.rs
index 055a25e..f497bc9 100644
--- a/src/vm/error.rs
+++ b/src/vm/error.rs
@@ -83,6 +83,10 @@ pub enum QueryExecutionError {
     IndeterminateOffset(#[from] time::error::IndeterminateOffset),
     #[error("Unable to determine local time zone")]
     TimeZoneLookupFailure(RcString),
+    #[error("Failed to compile regex: {0:?}")]
+    OnigurumaCompileError(#[from] onig::Error),
+    #[error("Invalid regex flag: {0}")]
+    InvalidRegexFlag(char),
     #[error("Input source gave an error")]
     InputError(#[from] InputError),
     #[error("{0:?}")]


Test patch:
diff --git a/tests/from_manual/mod.rs b/tests/from_manual/mod.rs
index 2c513a8..85e2e71 100644
--- a/tests/from_manual/mod.rs
+++ b/tests/from_manual/mod.rs
@@ -8,6 +8,7 @@ mod assignment;
 mod basic_filters;
 mod condition_and_comparison;
 mod functions;
+mod regex;
 mod string_format;
 mod time;
 mod types_and_values;
diff --git a/tests/from_manual/regex.rs b/tests/from_manual/regex.rs
new file mode 100644
index 0000000..3c165be
--- /dev/null
+++ b/tests/from_manual/regex.rs
@@ -0,0 +1,109 @@
+use crate::test;
+
+test!(
+    test1,
+    r#"
+    test("foo")
+    "#,
+    r#"
+    "foo"
+    "#,
+    r#"
+    true
+    "#
+);
+
+test!(
+    test2,
+    r#"
+    .[] | test("a b c # spaces are ignored"; "ix")
+    "#,
+    r#"
+    ["xabcd", "ABC"]
+    "#,
+    r#"
+    true
+    true
+    "#
+);
+
+test!(
+    match1,
+    r#"
+    match("(abc)+"; "g")
+    "#,
+    r#"
+    "abc abc"
+    "#,
+    r#"
+    {"offset": 0, "length": 3, "string": "abc", "captures": [{"offset": 0, "length": 3, "string": "abc", "name": null}]}
+    {"offset": 4, "length": 3, "string": "abc", "captures": [{"offset": 4, "length": 3, "string": "abc", "name": null}]}
+    "#
+);
+
+test!(
+    match2,
+    r#"
+    match("foo")
+    "#,
+    r#"
+    "foo bar foo"
+    "#,
+    r#"
+    {"offset": 0, "length": 3, "string": "foo", "captures": []}
+    "#
+);
+
+test!(
+    match3,
+    r#"
+    match(["foo", "ig"])
+    "#,
+    r#"
+    "foo bar FOO"
+    "#,
+    r#"
+    {"offset": 0, "length": 3, "string": "foo", "captures": []}
+    {"offset": 8, "length": 3, "string": "FOO", "captures": []}
+    "#
+);
+
+test!(
+    match4,
+    r#"
+    match("foo (?<bar123>bar)? foo"; "ig")
+    "#,
+    r#"
+    "foo bar foo foo  foo"
+    "#,
+    r#"
+    {"offset": 0, "length": 11, "string": "foo bar foo", "captures": [{"offset": 4, "length": 3, "string": "bar", "name": "bar123"}]}
+    {"offset": 12, "length": 8, "string": "foo  foo", "captures": [{"offset": -1, "length": 0, "string": null, "name": "bar123"}]}
+    "#
+);
+
+test!(
+    match5,
+    r#"
+    [ match("."; "g")] | length
+    "#,
+    r#"
+    "abc"
+    "#,
+    r#"
+    3
+    "#
+);
+
+test!(
+    capture1,
+    r#"
+    capture("(?<a>[a-z]+)-(?<n>[0-9]+)")
+    "#,
+    r#"
+    "xyzzy-14"
+    "#,
+    r#"
+    { "a": "xyzzy", "n": "14" }
+    "#
+);
diff --git a/tests/hand_written/mod.rs b/tests/hand_written/mod.rs
index da44901..5e31815 100644
--- a/tests/hand_written/mod.rs
+++ b/tests/hand_written/mod.rs
@@ -2,6 +2,7 @@ use crate::test;
 
 mod assignments;
 mod math;
+mod regex;
 
 test!(
     recursive,
diff --git a/tests/hand_written/regex.rs b/tests/hand_written/regex.rs
new file mode 100644
index 0000000..62ede4c
--- /dev/null
+++ b/tests/hand_written/regex.rs
@@ -0,0 +1,90 @@
+use crate::test;
+
+test!(
+    capture_multi_byte,
+    r#"
+    match("(.)"; "g")
+    "#,
+    r#"
+    "あaいうえお"
+    "#,
+    r#"
+    {"offset":0,"length":1,"string":"あ","captures":[{"offset":0,"length":1,"string":"あ","name":null}]}
+    {"offset":1,"length":1,"string":"a","captures":[{"offset":1,"length":1,"string":"a","name":null}]}
+    {"offset":2,"length":1,"string":"い","captures":[{"offset":2,"length":1,"string":"い","name":null}]}
+    {"offset":3,"length":1,"string":"う","captures":[{"offset":3,"length":1,"string":"う","name":null}]}
+    {"offset":4,"length":1,"string":"え","captures":[{"offset":4,"length":1,"string":"え","name":null}]}
+    {"offset":5,"length":1,"string":"お","captures":[{"offset":5,"length":1,"string":"お","name":null}]}
+    "#
+);
+
+test!(
+    capture_same_name,
+    r#"
+    match("(?<x>.)(?<x>.)"; "g") | .captures
+    "#,
+    r#"
+    "aあb"
+    "#,
+    r#"
+    [{"offset":0,"length":1,"string":"a","name":"x"},{"offset":1,"length":1,"string":"あ","name":"x"}]
+    "#
+);
+
+test!(
+    nonmatched_group,
+    r#"
+    match("う|(い)|(.)"; "g")
+    "#,
+    r#"
+    "あaいう"
+    "#,
+    r#"
+    {"offset":0,"length":1,"string":"あ","captures":[{"offset":-1,"string":null,"length":0,"name":null},{"offset":0,"length":1,"string":"あ","name":null}]}
+    {"offset":1,"length":1,"string":"a","captures":[{"offset":-1,"string":null,"length":0,"name":null},{"offset":1,"length":1,"string":"a","name":null}]}
+    {"offset":2,"length":1,"string":"い","captures":[{"offset":2,"length":1,"string":"い","name":null},{"offset":-1,"string":null,"length":0,"name":null}]}
+    {"offset":3,"length":1,"string":"う","captures":[{"offset":-1,"string":null,"length":0,"name":null},{"offset":-1,"string":null,"length":0,"name":null}]}
+    "#
+);
+
+test!(
+    capture_global,
+    r#"
+    capture("(?<a>.)-(?<b>.)(\\S*)"; "g")
+    "#,
+    r#"
+    "a-1 b-234 c"
+    "#,
+    r#"
+    {"a":"a","b":"1"}
+    {"a":"b","b":"2"}
+    "#
+);
+
+test!(
+    scan,
+    r#"
+    scan("(?<a>.)-(?<b>.)(\\S*)")
+    "#,
+    r#"
+    "a-1 b-234 c"
+    "#,
+    r#"
+    ["a","1",""]
+    ["b","2","34"]
+    "#
+);
+
+test!(
+    scan_no_capture,
+    r#"
+    scan("(?:.)-(?:.)(?:\\S*)")
+    "#,
+    r#"
+    "a-1 b-234 c"
+    "#,
+    r#"
+    "a-1"
+    "b-234"
+    "#
+);