Repository: shapesecurity/salvation (java)

Issue:
Error message for source-expression containing invalid characters should be verbous.
Ex. https://cspvalidator.org/#headerValue[]=script-src+%D0%B0&strategy=intersection . 
Say that chars outside ASCII 0x21-0x7E must be percent encoding or something like that

Pull request description:
fixes #90
 verbose error in tokeniser for bad characters in directive-value

Patch:
diff --git a/src/main/java/com/shapesecurity/salvation/Tokeniser.java b/src/main/java/com/shapesecurity/salvation/Tokeniser.java
index 6677f60..6ca196b 100644
--- a/src/main/java/com/shapesecurity/salvation/Tokeniser.java
+++ b/src/main/java/com/shapesecurity/salvation/Tokeniser.java
@@ -14,7 +14,7 @@ public class Tokeniser {
     private static final Pattern directiveSeparator = Pattern.compile(";");
     private static final Pattern policySeparator = Pattern.compile(",");
     private static final Pattern directiveNamePattern = Pattern.compile("[a-zA-Z0-9-]+");
-    private static final Pattern directiveValuePattern = Pattern.compile("[^;,\0- \\x7F]+");
+    private static final Pattern directiveValuePattern = Pattern.compile("[!-+--:<-~]+");
     @Nonnull protected final ArrayList<Token> tokens;
     @Nonnull protected final String sourceText;
     protected final int length;
@@ -98,7 +98,9 @@ public class Tokeniser {
                 continue;
             while (this.hasNext()) {
                 if (!this.eatDirectiveValue()) {
-                    throw this.createError("expecting directive-value but found " + this.next());
+                    String token = this.next();
+                    int cp = token.codePointAt(0);
+                    throw this.createError(String.format("expecting directive-value but found U+%04X (%s). Non-ASCII and non-printable characters must be percent-encoded", cp, new String(new int[]{cp}, 0, 1)));
                 }
                 if (this.eatSeparator())
                     break;


Test patch:
diff --git a/src/test/java/com/shapesecurity/salvation/ParserTest.java b/src/test/java/com/shapesecurity/salvation/ParserTest.java
index bb2e6b0..35a8b7a 100644
--- a/src/test/java/com/shapesecurity/salvation/ParserTest.java
+++ b/src/test/java/com/shapesecurity/salvation/ParserTest.java
@@ -405,6 +405,38 @@ public class ParserTest extends CSPTest {
             parse("img-src example.com 'unsafe-redirect'").getDirectiveByType(ImgSrcDirective.class).show());
     }
 
+    @Test
+    public void testTokeniserErrors() throws ParseException, TokeniserException {
+        try {
+            parse("script-src abц"); // non-ASCII char in source-expression
+            fail();
+        } catch (TokeniserException e1) {
+            assertEquals("expecting directive-value but found U+0446 (ц). Non-ASCII and non-printable characters must be percent-encoded", e1.getMessage());
+        }
+
+        try {
+            parse("script-src 𝌆"); // non-ASCII char in source-expression
+            fail();
+        } catch (TokeniserException e1) {
+            assertEquals("expecting directive-value but found U+1D306 (𝌆). Non-ASCII and non-printable characters must be percent-encoded", e1.getMessage());
+        }
+
+
+        try {
+            parse("script-src http://abc.com/ш"); // non-ASCII char in source-expression
+            fail();
+        } catch (TokeniserException e1) {
+            assertEquals("expecting directive-value but found U+0448 (ш). Non-ASCII and non-printable characters must be percent-encoded", e1.getMessage());
+        }
+
+        try {
+            parse("plugin-types х/п"); // non-ASCII char in source-expression
+            fail();
+        } catch (TokeniserException e1) {
+            assertEquals("expecting directive-value but found U+0445 (х). Non-ASCII and non-printable characters must be percent-encoded", e1.getMessage());
+        }
+    }
+
 
     @Test
     public void testRealData()