Repository: apache/jena (java)

Issue:
Turtle parser: Carriage returns in literals delimited by a single quote (single or double) are not flagged. 
### Version

5.2.0

### What happened?

A literal in turtle in a single quoted section is not allowed to have a [carriage return in them](https://www.w3.org/TR/turtle/#turtle-literals).
Validation should detect such cases and report errors.

### Relevant output and stacktrace

_No response_

### Are you interested in making a pull request?

None

Pull request description:
GH-2826: Disallow raw CR in single quoted string
GitHub issue resolved #2826

----

 - [x] Tests are included.
 - [x] Key commit messages start with the issue number (GH-xxxx)

By submitting this pull request, I acknowledge that I am making a contribution to the Apache Software Foundation under the terms and conditions of the [Contributor's Agreement](https://www.apache.org/licenses/contributor-agreements.html).


Patch:
diff --git a/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerText.java b/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerText.java
index bc7de737c2..0f046bd583 100644
--- a/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerText.java
+++ b/jena-arq/src/main/java/org/apache/jena/riot/tokens/TokenizerText.java
@@ -215,7 +215,7 @@ public final class TokenizerText implements Tokenizer
                 int ch3 = reader.peekChar();
                 if ( ch3 == ch ) {
                     reader.readChar();     // Read potential third quote.
-                    token.setImage(readLongString(ch, false));
+                    token.setImage(readStringQuote3(ch, false));
                     StringType st = (ch == CH_QUOTE1) ? StringType.LONG_STRING1 : StringType.LONG_STRING2;
                     token.setStringType(st);
                 } else {
@@ -232,7 +232,7 @@ public final class TokenizerText implements Tokenizer
                 }
             } else {
                 // One quote character.
-                token.setImage(readString(ch, ch));
+                token.setImage(readStringQuote1(ch, ch));
                 // Record exactly what form of STRING was seen.
                 StringType st = (ch == CH_QUOTE1) ? StringType.STRING1 : StringType.STRING2;
                 token.setStringType(st);
@@ -788,7 +788,7 @@ public final class TokenizerText implements Tokenizer
 
     // Get characters between two markers.
     // strEscapes may be processed
-    private String readString(int startCh, int endCh) {
+    private String readStringQuote1(int startCh, int endCh) {
         // Position at start of string.
         stringBuilder.setLength(0);
         // Assumes first delimiter char read already.
@@ -802,15 +802,20 @@ public final class TokenizerText implements Tokenizer
                     warning("Unicode replacement character U+FFFD in string");
             }
             if ( ch == NotACharacter || ch == ReverseOrderBOM )
-                warning("Unicode non-character U+%4X in string", ch);
-            if ( ch == EOF ) {
-                // if ( endNL ) return stringBuilder.toString();
+                warning("Unicode non-character U+%04X in string", ch);
+            if ( ch == EOF )
                 fatal("Broken token: %s", stringBuilder.toString());
-            }
-            else if ( ch == NL )
-                fatal("Broken token (newline): %s", stringBuilder.toString());
             else if ( ch == endCh )
                 return stringBuilder.toString();
+            else if ( ch == NL )
+                fatal("Broken token (newline in string)", stringBuilder.toString());
+            else if ( ch == CR )
+                fatal("Broken token (carriage return in string)", stringBuilder.toString());
+            // Legal in Turtle/N-Triples - maybe warn?
+//            else if ( ch == FF )
+//                warning("Bad token (form feed in string)", stringBuilder.toString());
+//            else if ( ch == VT )
+//                fatal("Bad token (vertical tab in string)", stringBuilder.toString());
             else if ( ch == CH_RSLASH )
                 // Allow escaped replacement character.
                 ch = readLiteralEscape();
@@ -818,7 +823,7 @@ public final class TokenizerText implements Tokenizer
         }
     }
 
-    private String readLongString(int quoteChar, boolean endNL) {
+    private String readStringQuote3(int quoteChar, boolean endNL) {
         stringBuilder.setLength(0);
         for (;;) {
             int ch = reader.readChar();
diff --git a/jena-base/src/main/java/org/apache/jena/atlas/lib/Chars.java b/jena-base/src/main/java/org/apache/jena/atlas/lib/Chars.java
index e9db62c855..fde9bb8d11 100644
--- a/jena-base/src/main/java/org/apache/jena/atlas/lib/Chars.java
+++ b/jena-base/src/main/java/org/apache/jena/atlas/lib/Chars.java
@@ -177,20 +177,22 @@ public class Chars
     // REPLACEMENT CHARACTER
     public static final char REPLACEMENT     = 0xFFFD ;
 
-    // "Not a character" - a Unicode noncharacter codepoint that is not legal in UTF-8.
+    // "Not a character" - a Unicode non-character codepoint that is not legal in UTF-8.
     public static final char NotACharacter   = 0xFFFF ;
-    // Detect byte order by contrast (BOM reversed) - a Unicode noncharacter codepoint that is not legal in UTF-8.
+    // Detect byte order by contrast (BOM reversed) - a Unicode non-character codepoint that is not legal in UTF-8.
     public static final char ReverseOrderBOM = 0xFFFE ;
 
     /** Undefined character (exact meaning depends on use) - not a Unicode codepoint */
     public static final int  UNSET           =  -2 ;
-    public static final char NL              = '\n' ;
-    public static final char LF              = NL ;     // Alt name.
-    public static final char CR              = '\r' ;
-    public static final char TAB             = '\t' ;
-    public static final char FF              = '\f' ;   // Form feed
-    public static final char SPC             = ' ' ;
-    public static final char BSPACE          = '\b' ;
+    public static final char NL              = '\n' ;       // U+000A
+    public static final char LF              = NL ;         // U+000A - alternative name
+    public static final char CR              = '\r' ;       // U+000D
+    public static final char TAB             = '\t' ;       // U+0009
+    public static final char FF              = '\f' ;       // U+000B - Form feed
+    public static final char VT              = '\u000B' ;   // U+000C - Vertical tab
+    public static final char SPC             = ' ' ;        // U+0020
+    public static final char BSPACE          = '\b' ;       // U+0008 - Backspace
+    public static final char DEL             = '\u007F' ;   // U+0008 - Delete
 
     public static final char CH_ZERO         =  (char)0 ;
 
@@ -231,9 +233,13 @@ public class Chars
     public static final char CH_VBAR         = '|' ;
     public static final char CH_TILDE        = '~' ;
 
-    // Byte versions of the above
+    // Byte versions of some of the above
     public static final byte B_NL            = NL ;
+    public static final byte B_LF            = LF ;
     public static final byte B_CR            = CR ;
+    public static final byte B_FF            = FF ;
+    public static final byte B_VT            = VT ;
+    public static final byte B_SPC           = SPC ;
 
     public static final byte B_LBRACKET      = '[' ;
     public static final byte B_RBRACKET      = ']' ;


Test patch:
diff --git a/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerText.java b/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerText.java
index 5ada5cc3b0..b4c2865afb 100644
--- a/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerText.java
+++ b/jena-arq/src/test/java/org/apache/jena/riot/tokens/TestTokenizerText.java
@@ -313,6 +313,38 @@ public class TestTokenizerText {
         tokenizeAndTestExact("'\\U00000020'", StringType.STRING1, " ");
     }
 
+    // Raw newline and carriage return not allowed in single quoted strings
+
+    @Test(expected = RiotParseException.class)
+    public void tokenUnit_str12() {
+        tokenFirst("'abc\rdef'");
+    }
+
+    @Test(expected = RiotParseException.class)
+    public void tokenUnit_str13() {
+        tokenFirst("'abc\ndef'");
+    }
+
+    @Test(expected = RiotParseException.class)
+    public void tokenUnit_str14() {
+        tokenFirst("\"abc\rdef\"");
+    }
+
+    @Test(expected = RiotParseException.class)
+    public void tokenUnit_str15() {
+        tokenFirst("\"abc\ndef\"");
+    }
+
+    @Test(expected = RiotParseException.class)
+    public void tokenUnit_str16() {
+        tokenFirst("'\r'");
+    }
+
+    @Test(expected = RiotParseException.class)
+    public void tokenUnit_str17() {
+        tokenFirst("\"\n\"");
+    }
+
     @Test
     public void tokenUnit_str_long1() {
         tokenizeAndTestExact("'''aaa'''", StringType.LONG_STRING1, "aaa");
@@ -368,6 +400,27 @@ public class TestTokenizerText {
         tokenFirst("'''");
     }
 
+    @Test
+    public void tokenUnit_str_long20() {
+        tokenizeAndTestExact("\"\"\"abc\ndef\"\"\"", StringType.LONG_STRING2, "abc\ndef");
+    }
+
+    @Test
+    public void tokenUnit_str_long21() {
+        tokenizeAndTestExact("\"\"\"abc\rdef\"\"\"", StringType.LONG_STRING2, "abc\rdef");
+    }
+
+    @Test
+    public void tokenUnit_str_long22() {
+        tokenizeAndTestExact("'''abc\ndef'''", StringType.LONG_STRING1, "abc\ndef");
+    }
+
+    @Test
+    public void tokenUnit_str_long23() {
+        tokenizeAndTestExact("'''abc\rdef'''", StringType.LONG_STRING1, "abc\rdef");
+    }
+
+
     @Test
     public void tokenUnit_str_long12() {
         tokenizeAndTestExact("'''x'''@en", TokenType.LITERAL_LANG, "x", "en");