Repository: networknt/json-schema-validator
Pull request #1049 (2024-06-06 12:49:40)

Detect invalid escape for ecmascript
This fixes the test failure from the JSON Schema Test Suite in `optional/ecmascript-regex.json` as `joni` does not detect invalid escapes.

This makes the following changes
* The `regex` format has been changed to also follow the configuration of whether to use `ecma-262` compliant regex. Previously it always used the `jdk` regex.
* The `JoniRegularExpression` implementation uses a regex to detect if an invalid escape is used as `joni` doesn't detect invalid escapes. In actuality it's upstream `Oniguruma` doesn't actually support ecma as a syntax.
* The latest tests from the JSON Schema Test Suite for `optional/ecmascript-regex.json` has been updated. In actuality it doesn't change anything but it is clearer that `"format": "regex"` is used in the test
* The test harness now enables format assertions when running the `optional/ecmascript-regex.json` test


codecov-commenter: ## [Codecov](https://app.codecov.io/gh/networknt/json-schema-validator/pull/1049?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=networknt) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 78.74%. Comparing base [(`48ca3c2`)](https://app.codecov.io/gh/networknt/json-schema-validator/commit/48ca3c22995e32797e9defdb8b77869f0d15caf3?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=networknt) to head [(`e11488a`)](https://app.codecov.io/gh/networknt/json-schema-validator/commit/e11488af30d485a733f4323ac3e0c8185a65b9fc?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=networknt).
> Report is 19 commits behind head on master.


:exclamation: Your organization needs to install the [Codecov GitHub app](https://github.com/apps/codecov/installations/select_target) to enable full functionality.

<details><summary>Additional details and impacted files</summary>


```diff
@@             Coverage Diff              @@
##             master    #1049      +/-   ##
============================================
- Coverage     78.90%   78.74%   -0.16%     
- Complexity     1965     2033      +68     
============================================
  Files           172      191      +19     
  Lines          6352     6596     +244     
  Branches       1255     1260       +5     
============================================
+ Hits           5012     5194     +182     
- Misses          867      927      +60     
- Partials        473      475       +2     
```



</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/networknt/json-schema-validator/pull/1049?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=networknt).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=networknt).


Patch:
diff --git a/src/main/java/com/networknt/schema/format/RegexFormat.java b/src/main/java/com/networknt/schema/format/RegexFormat.java
index c6f778b..191e636 100644
--- a/src/main/java/com/networknt/schema/format/RegexFormat.java
+++ b/src/main/java/com/networknt/schema/format/RegexFormat.java
@@ -13,24 +13,22 @@
  */
 package com.networknt.schema.format;
 
-import java.util.regex.Pattern;
-import java.util.regex.PatternSyntaxException;
-
 import com.networknt.schema.ExecutionContext;
 import com.networknt.schema.Format;
+import com.networknt.schema.ValidationContext;
+import com.networknt.schema.regex.RegularExpression;
 
 /**
  * Format for regex.
  */
 public class RegexFormat implements Format {
     @Override
-    public boolean matches(ExecutionContext executionContext, String value) {
+    public boolean matches(ExecutionContext executionContext, ValidationContext validationContext, String value) {
         if (null == value) return true;
         try {
-            Pattern.compile(value);
+            RegularExpression.compile(value, validationContext);
             return true;
-
-        } catch (PatternSyntaxException e) {
+        } catch (RuntimeException e) {
             return false;
         }
     }
diff --git a/src/main/java/com/networknt/schema/format/UriFormat.java b/src/main/java/com/networknt/schema/format/UriFormat.java
index 81d9034..1b98c02 100644
--- a/src/main/java/com/networknt/schema/format/UriFormat.java
+++ b/src/main/java/com/networknt/schema/format/UriFormat.java
@@ -13,7 +13,7 @@ public class UriFormat extends AbstractRFC3986Format {
             // Java URI accepts non ASCII characters and this is not a valid in RFC3986
             result = uri.toString().codePoints().allMatch(ch -> ch < 0x7F);
             if (result) {
-                String query = uri.getQuery();
+                String query = uri.getRawQuery();
                 if (query != null) {
                     // [ and ] must be percent encoded
                     if (query.indexOf('[') != -1 || query.indexOf(']') != -1) {
diff --git a/src/main/java/com/networknt/schema/format/UriReferenceFormat.java b/src/main/java/com/networknt/schema/format/UriReferenceFormat.java
index d48650f..4cb5a20 100644
--- a/src/main/java/com/networknt/schema/format/UriReferenceFormat.java
+++ b/src/main/java/com/networknt/schema/format/UriReferenceFormat.java
@@ -11,7 +11,7 @@ public class UriReferenceFormat extends AbstractRFC3986Format {
         // Java URI accepts non ASCII characters and this is not a valid in RFC3986
         boolean result = uri.toString().codePoints().allMatch(ch -> ch < 0x7F);
         if (result) {
-            String query = uri.getQuery();
+            String query = uri.getRawQuery();
             if (query != null) {
                 // [ and ] must be percent encoded
                 if (query.indexOf('[') != -1 || query.indexOf(']') != -1) {
diff --git a/src/main/java/com/networknt/schema/regex/JDKRegularExpression.java b/src/main/java/com/networknt/schema/regex/JDKRegularExpression.java
index 6be0c7e..408117f 100644
--- a/src/main/java/com/networknt/schema/regex/JDKRegularExpression.java
+++ b/src/main/java/com/networknt/schema/regex/JDKRegularExpression.java
@@ -2,6 +2,9 @@ package com.networknt.schema.regex;
 
 import java.util.regex.Pattern;
 
+/**
+ * JDK {@link RegularExpression}. 
+ */
 class JDKRegularExpression implements RegularExpression {
     private final Pattern pattern;
 
diff --git a/src/main/java/com/networknt/schema/regex/JoniRegularExpression.java b/src/main/java/com/networknt/schema/regex/JoniRegularExpression.java
index bad4251..5e205e4 100644
--- a/src/main/java/com/networknt/schema/regex/JoniRegularExpression.java
+++ b/src/main/java/com/networknt/schema/regex/JoniRegularExpression.java
@@ -1,16 +1,28 @@
 package com.networknt.schema.regex;
 
 import java.nio.charset.StandardCharsets;
+import java.util.regex.Pattern;
 
 import org.jcodings.specific.UTF8Encoding;
 import org.joni.Option;
 import org.joni.Regex;
 import org.joni.Syntax;
+import org.joni.exception.SyntaxException;
 
+/**
+ * ECMAScript {@link RegularExpression}.
+ */
 class JoniRegularExpression implements RegularExpression {
     private final Regex pattern;
+    private final Pattern INVALID_ESCAPE_PATTERN = Pattern.compile(
+            ".*\\\\([aeg-moqyzACE-OQ-RT-VX-Z1-9]|c$|[pP]([^{]|$)|u([^{0-9]|$)|x([0-9a-fA-F][^0-9a-fA-F]|[^0-9a-fA-F][0-9a-fA-F]|[^0-9a-fA-F][^0-9a-fA-F]|.?$)).*");
 
     JoniRegularExpression(String regex) {
+        this(regex, Syntax.ECMAScript);
+    }
+
+    JoniRegularExpression(String regex, Syntax syntax) {
+        validate(regex);
         // Joni is too liberal on some constructs
         String s = regex
             .replace("\\d", "[0-9]")
@@ -21,7 +33,19 @@ class JoniRegularExpression implements RegularExpression {
             .replace("\\S", "[^ \\f\\n\\r\\t\\v\\u00a0\\u1680\\u2000-\\u200a\\u2028\\u2029\\u202f\\u205f\\u3000\\ufeff]");
 
         byte[] bytes = s.getBytes(StandardCharsets.UTF_8);
-        this.pattern = new Regex(bytes, 0, bytes.length, Option.SINGLELINE, UTF8Encoding.INSTANCE, Syntax.ECMAScript);
+        this.pattern = new Regex(bytes, 0, bytes.length, Option.SINGLELINE, UTF8Encoding.INSTANCE, syntax);
+    }
+
+    protected void validate(String regex) {
+        // Joni is not strict with escapes
+        if (INVALID_ESCAPE_PATTERN.matcher(regex).matches()) {
+            /*
+             * One option considered was a custom Encoding implementation that rejects
+             * certain code points but it is unable to distinguish \a vs \cG for instance as
+             * both translate to BEL
+             */
+            throw new SyntaxException("Invalid escape");
+        }
     }
 
     @Override
diff --git a/src/main/java/com/networknt/schema/regex/RegularExpression.java b/src/main/java/com/networknt/schema/regex/RegularExpression.java
index 64c1ba2..83f4d22 100644
--- a/src/main/java/com/networknt/schema/regex/RegularExpression.java
+++ b/src/main/java/com/networknt/schema/regex/RegularExpression.java
@@ -2,6 +2,9 @@ package com.networknt.schema.regex;
 
 import com.networknt.schema.ValidationContext;
 
+/**
+ * Regular expression.
+ */
 @FunctionalInterface
 public interface RegularExpression {
     boolean matches(String value);


Test patch:
diff --git a/src/test/java/com/networknt/schema/AbstractJsonSchemaTestSuite.java b/src/test/java/com/networknt/schema/AbstractJsonSchemaTestSuite.java
index 66a6856..3852f63 100644
--- a/src/test/java/com/networknt/schema/AbstractJsonSchemaTestSuite.java
+++ b/src/test/java/com/networknt/schema/AbstractJsonSchemaTestSuite.java
@@ -55,7 +55,9 @@ public abstract class AbstractJsonSchemaTestSuite extends HTTPServiceSupport {
 
     private static void executeTest(JsonSchema schema, TestSpec testSpec) {
         Set<ValidationMessage> errors = schema.validate(testSpec.getData(), OutputFormat.DEFAULT, (executionContext, validationContext) -> {
-            if (testSpec.getTestCase().getSource().getPath().getParent().toString().endsWith("format")) {
+            if (testSpec.getTestCase().getSource().getPath().getParent().toString().endsWith("format")
+                    || "ecmascript-regex.json"
+                            .equals(testSpec.getTestCase().getSource().getPath().getFileName().toString())) {
                 executionContext.getExecutionConfig().setFormatAssertionsEnabled(true);
             }
         });
diff --git a/src/test/java/com/networknt/schema/format/UriFormatTest.java b/src/test/java/com/networknt/schema/format/UriFormatTest.java
index 52a850e..0cac5ff 100644
--- a/src/test/java/com/networknt/schema/format/UriFormatTest.java
+++ b/src/test/java/com/networknt/schema/format/UriFormatTest.java
@@ -56,7 +56,21 @@ class UriFormatTest {
                 InputFormat.JSON);
         assertFalse(messages.isEmpty());
     }
-    
+
+    @Test
+    void queryWithEncodedBracketsShouldPass() {
+        String schemaData = "{\r\n"
+                + "  \"format\": \"uri\"\r\n"
+                + "}";
+
+        SchemaValidatorsConfig config = new SchemaValidatorsConfig();
+        config.setFormatAssertionsEnabled(true);
+        JsonSchema schema = JsonSchemaFactory.getInstance(VersionFlag.V202012).getSchema(schemaData, config);
+        Set<ValidationMessage> messages = schema.validate("\"https://test.com/assets/product.pdf?filter%5Btest%5D=1\"",
+                InputFormat.JSON);
+        assertTrue(messages.isEmpty());
+    }
+
     @Test
     void iriShouldFail() {
         String schemaData = "{\r\n"
@@ -70,5 +84,4 @@ class UriFormatTest {
                 InputFormat.JSON);
         assertFalse(messages.isEmpty());
     }
-
 }
diff --git a/src/test/java/com/networknt/schema/format/UriReferenceFormatTest.java b/src/test/java/com/networknt/schema/format/UriReferenceFormatTest.java
index 787ba65..0f0e9aa 100644
--- a/src/test/java/com/networknt/schema/format/UriReferenceFormatTest.java
+++ b/src/test/java/com/networknt/schema/format/UriReferenceFormatTest.java
@@ -57,6 +57,20 @@ class UriReferenceFormatTest {
         assertFalse(messages.isEmpty());
     }
 
+    @Test
+    void queryWithEncodedBracketsShouldPass() {
+        String schemaData = "{\r\n"
+                + "  \"format\": \"uri-reference\"\r\n"
+                + "}";
+
+        SchemaValidatorsConfig config = new SchemaValidatorsConfig();
+        config.setFormatAssertionsEnabled(true);
+        JsonSchema schema = JsonSchemaFactory.getInstance(VersionFlag.V202012).getSchema(schemaData, config);
+        Set<ValidationMessage> messages = schema.validate("\"https://test.com/assets/product.pdf?filter%5Btest%5D=1\"",
+                InputFormat.JSON);
+        assertTrue(messages.isEmpty());
+    }
+
     @Test
     void iriShouldFail() {
         String schemaData = "{\r\n"
diff --git a/src/test/java/com/networknt/schema/regex/JoniRegularExpressionTest.java b/src/test/java/com/networknt/schema/regex/JoniRegularExpressionTest.java
new file mode 100644
index 0000000..fc1f899
--- /dev/null
+++ b/src/test/java/com/networknt/schema/regex/JoniRegularExpressionTest.java
@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) 2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.networknt.schema.regex;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+import org.joni.exception.SyntaxException;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+/**
+ * Tests for JoniRegularExpression.
+ */
+class JoniRegularExpressionTest {
+
+    enum InvalidEscapeInput {
+        A("\\a"),
+        HELLOA("hello\\a"),
+        C("\\c"),
+        E("\\e"),
+        G("\\g"),
+        H("\\h"),
+        I("\\i"),
+        J("\\j"),
+        K("\\k"),
+        L("\\l"),
+        M("\\m"),
+        O("\\o"),
+        Q("\\q"),
+        U("\\u"),
+        X("\\x"),
+        X1("\\x1"),
+        XGG("\\xgg"),
+        X1G("\\x1g"),
+        Y("\\y"),
+        Z("\\z"),
+        _1("\\1"),
+        _2("\\2"),
+        _3("\\3"),
+        _4("\\4"),
+        _5("\\5"),
+        _6("\\6"),
+        _7("\\7"),
+        _8("\\8"),
+        _9("\\9");
+
+        String value;
+
+        InvalidEscapeInput(String value) {
+            this.value = value;
+        }
+    }
+
+    @ParameterizedTest
+    @EnumSource(InvalidEscapeInput.class)
+    void invalidEscape(InvalidEscapeInput input) {
+        SyntaxException e = assertThrows(SyntaxException.class, () -> new JoniRegularExpression(input.value));
+        assertEquals("Invalid escape", e.getMessage());
+    }
+
+    enum ValidEscapeInput {
+        B("\\b"),
+        D("\\d"),
+        CAP_D("\\D"),
+        W("\\w"),
+        CAP_W("\\W"),
+        S("\\s"),
+        CAP_S("\\S"),
+        T("\\t"),
+        U1234("\\u1234"),
+        R("\\r"),
+        N("\\n"),
+        V("\\v"),
+        F("\\f"),
+        X12("\\x12"),
+        X1F("\\x1f"),
+        X1234("\\x1234"),
+        P("\\p{Letter}cole"), // unicode property
+        CAP_P("\\P{Letter}cole"), // unicode property
+        _0("\\0"),
+        CA("\\cA"), // control
+        CB("\\cB"), // control
+        CC("\\cC"), // control
+        CG("\\cG"); // control
+
+        String value;
+
+        ValidEscapeInput(String value) {
+            this.value = value;
+        }
+    }
+
+    @ParameterizedTest
+    @EnumSource(ValidEscapeInput.class)
+    void validEscape(ValidEscapeInput input) {
+        assertDoesNotThrow(() -> new JoniRegularExpression(input.value));
+    }
+    
+    @Test
+    void invalidPropertyName() {
+        assertThrows(SyntaxException.class, () -> new JoniRegularExpression("\\p"));
+        assertThrows(SyntaxException.class, () -> new JoniRegularExpression("\\P"));
+        assertThrows(SyntaxException.class, () -> new JoniRegularExpression("\\pa"));
+        assertThrows(SyntaxException.class, () -> new JoniRegularExpression("\\Pa"));
+    }
+}
diff --git a/src/test/suite/tests/draft-next/optional/ecmascript-regex.json b/src/test/suite/tests/draft-next/optional/ecmascript-regex.json
index 2721145..c96d4e0 100644
--- a/src/test/suite/tests/draft-next/optional/ecmascript-regex.json
+++ b/src/test/suite/tests/draft-next/optional/ecmascript-regex.json
@@ -409,12 +409,12 @@
         "description": "\\a is not an ECMA 262 control escape",
         "schema": {
             "$schema": "https://json-schema.org/draft/next/schema",
-            "$ref": "https://json-schema.org/draft/next/schema"
+            "format": "regex"
         },
         "tests": [
             {
                 "description": "when used as a pattern",
-                "data": { "pattern": "\\a" },
+                "data": "\\a",
                 "valid": false
             }
         ]
diff --git a/src/test/suite/tests/draft2020-12/optional/ecmascript-regex.json b/src/test/suite/tests/draft2020-12/optional/ecmascript-regex.json
index 41eaafe..9764953 100644
--- a/src/test/suite/tests/draft2020-12/optional/ecmascript-regex.json
+++ b/src/test/suite/tests/draft2020-12/optional/ecmascript-regex.json
@@ -409,15 +409,13 @@
         "description": "\\a is not an ECMA 262 control escape",
         "schema": {
             "$schema": "https://json-schema.org/draft/2020-12/schema",
-            "$ref": "https://json-schema.org/draft/2020-12/schema"
+            "format": "regex"
         },
         "tests": [
             {
                 "description": "when used as a pattern",
-                "data": { "pattern": "\\a" },
-                "valid": false,
-                "disabled": true,
-                "reason": "TODO: RegexFormat does not support ECMA 262 regular expressions"
+                "data": "\\a",
+                "valid": false
             }
         ]
     },