Repository: checkstyle/checkstyle

Problem statement:
Javadoc parser/grammar doesn't report the correct lines in Javadoc AST.
Identified at https://github.com/checkstyle-antlr4/checkstyle-antlr4/issues/15, ANTLR doesn't support the `\r` (pre-OSX macos) line ending any longer, and has no plans to add support. Therefore, when parsing a Javadoc in a file that uses the `\r` line ending, the line number never increases in the Javadoc:

```java

public  class Test {
    /**
     * This is a javadoc comment.
     * More info, on another line.
     */
    void doStuff() {

    }
}


```

Unix utilities do not play well with `\r`, which renders as `^M` with `cat`:

```bash

➜  src cat -v Test.java 
public  class Test {^M    /**^M     * This is a javadoc comment.^M     * More info, on another line.^M     */^M    void doStuff() {^M^M    }^M}^M%

➜  src java -jar checkstyle-8.44-all.jar -J Test.java
CLASS_DEF -> CLASS_DEF [1:0]
|--MODIFIERS -> MODIFIERS [1:0]
|   `--LITERAL_PUBLIC -> public [1:0]
|--LITERAL_CLASS -> class [1:8]
|--IDENT -> Test [1:14]
`--OBJBLOCK -> OBJBLOCK [1:19]
    |--LCURLY -> { [1:19]
    |--METHOD_DEF -> METHOD_DEF [6:4]
    |   |--MODIFIERS -> MODIFIERS [6:4]
    |   |--TYPE -> TYPE [6:4]
    |   |   |--BLOCK_COMMENT_BEGIN -> /* [2:4]
    |   |   |   |--COMMENT_CONTENT -> *\r     * This is a javadoc comment.\r     * More info, on another line.\r      [2:6]
    |   |   |   |   `--JAVADOC -> JAVADOC [2:7] // ok
    |   |   |   |       |--NEWLINE -> \r [2:7] // ok
    |   |   |   |       |--LEADING_ASTERISK ->      * [2:8] // wrong, should be 3
    |   |   |   |       |--TEXT ->  This is a javadoc comment. [2:14] // wrong, should be 3
    |   |   |   |       |--NEWLINE -> \r [2:41] // wrong, should be 3
    |   |   |   |       |--LEADING_ASTERISK ->      * [2:42]  // wrong, should be 4
    |   |   |   |       |--TEXT ->  More info, on another line. [2:48]  // wrong, should be 4
    |   |   |   |       |--NEWLINE -> \r [2:76]  // wrong, should be 4
    |   |   |   |       |--TEXT ->       [2:77] // wrong, should be 5
    |   |   |   |       `--EOF -> <EOF> [2:82] // wrong, should be 5
    |   |   |   `--BLOCK_COMMENT_END -> */ [5:5]
    |   |   `--LITERAL_VOID -> void [6:4]
    |   |--IDENT -> doStuff [6:9]
    |   |--LPAREN -> ( [6:16]
    |   |--PARAMETERS -> PARAMETERS [6:17]
    |   |--RPAREN -> ) [6:17]
    |   `--SLIST -> { [6:19]
    |       `--RCURLY -> } [8:4]
    `--RCURLY -> } [9:0]

```

As you can see, the Javadoc starts on line 2, but then the line number never advances within that Javadoc AST. This is because the `consume()` function in ANTLR's interpreter doesn't recognize the `\r` alone as a line ending any longer (it only looks for `\n`, which covers the other two line endings). We should be able to handle any line ending that Java supports, which is `\r`, `\n`, and `\r\n`. 

I would expect the Javadoc AST to track line numbers correctly. Similar issue was originally reported at https://github.com/checkstyle/checkstyle/issues/2329, and problem was also noted at https://github.com/checkstyle/checkstyle/issues/2329#issuecomment-311235254.


Hints:

Patch:
diff --git a/src/main/java/com/puppycrawl/tools/checkstyle/JavadocDetailNodeParser.java b/src/main/java/com/puppycrawl/tools/checkstyle/JavadocDetailNodeParser.java
--- a/src/main/java/com/puppycrawl/tools/checkstyle/JavadocDetailNodeParser.java
+++ b/src/main/java/com/puppycrawl/tools/checkstyle/JavadocDetailNodeParser.java
@@ -166,7 +166,7 @@ public ParseStatus parseJavadocAsDetailNode(DetailAST javadocCommentAst) {
      */
     private static JavadocParser createJavadocParser(String blockComment,
             DescriptiveErrorListener errorListener) {
-        final JavadocLexer lexer = new JavadocLexer(CharStreams.fromString(blockComment));
+        final JavadocLexer lexer = new JavadocLexer(CharStreams.fromString(blockComment), true);
 
         final CommonTokenStream tokens = new CommonTokenStream(lexer);
 
diff --git a/src/main/java/com/puppycrawl/tools/checkstyle/grammar/CrAwareLexerSimulator.java b/src/main/java/com/puppycrawl/tools/checkstyle/grammar/CrAwareLexerSimulator.java
new file mode 100644
--- /dev/null
+++ b/src/main/java/com/puppycrawl/tools/checkstyle/grammar/CrAwareLexerSimulator.java
@@ -0,0 +1,77 @@
+////////////////////////////////////////////////////////////////////////////////
+// checkstyle: Checks Java source code for adherence to a set of rules.
+// Copyright (C) 2001-2021 the original author or authors.
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+////////////////////////////////////////////////////////////////////////////////
+
+package com.puppycrawl.tools.checkstyle.grammar;
+
+import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.Lexer;
+import org.antlr.v4.runtime.atn.ATN;
+import org.antlr.v4.runtime.atn.LexerATNSimulator;
+import org.antlr.v4.runtime.atn.PredictionContextCache;
+import org.antlr.v4.runtime.dfa.DFA;
+
+/**
+ * Extends the LexerATNSimulator class in order to override
+ * the 'consume()' method so that we can handle '\r' line
+ * endings (pre-OSX macOs line endings) correctly in the
+ * ANTLR lexer.
+ */
+public class CrAwareLexerSimulator extends LexerATNSimulator {
+
+    /**
+     * Constructs a CrAwareLexerSimulator to interpret the input
+     * from the lexer.
+     *
+     * @param lexer the current lexer
+     * @param augmented the augmented transition network
+     * @param decisionToDfa the DFA to store our states in
+     * @param sharedContextCache cache of PredictionContext objects
+     */
+    public CrAwareLexerSimulator(Lexer lexer, ATN augmented,
+                                 DFA[] decisionToDfa,
+                                 PredictionContextCache sharedContextCache) {
+        super(lexer, augmented, decisionToDfa, sharedContextCache);
+    }
+
+    /**
+     * Overrides the 'consume()' method to add support for
+     * '\r' (carriage return) line endings.
+     *
+     * @param input the Character stream of the file we are parsing
+     */
+    @Override
+    public void consume(CharStream input) {
+        final int currentChar = input.LA(1);
+        if (currentChar == '\n') {
+            line++;
+            charPositionInLine = 0;
+        }
+        else if (currentChar == '\r') {
+            final int nextChar = input.LA(2);
+            if (nextChar != '\n') {
+                line++;
+                charPositionInLine = 0;
+            }
+        }
+        else {
+            charPositionInLine++;
+        }
+        input.consume();
+    }
+}


Test patch:
diff --git a/src/test/java/com/puppycrawl/tools/checkstyle/grammar/CrAwareLexerTest.java b/src/test/java/com/puppycrawl/tools/checkstyle/grammar/CrAwareLexerTest.java
new file mode 100644
--- /dev/null
+++ b/src/test/java/com/puppycrawl/tools/checkstyle/grammar/CrAwareLexerTest.java
@@ -0,0 +1,83 @@
+////////////////////////////////////////////////////////////////////////////////
+// checkstyle: Checks Java source code for adherence to a set of rules.
+// Copyright (C) 2001-2021 the original author or authors.
+//
+// This library is free software; you can redistribute it and/or
+// modify it under the terms of the GNU Lesser General Public
+// License as published by the Free Software Foundation; either
+// version 2.1 of the License, or (at your option) any later version.
+//
+// This library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+// Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public
+// License along with this library; if not, write to the Free Software
+// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+////////////////////////////////////////////////////////////////////////////////
+
+package com.puppycrawl.tools.checkstyle.grammar;
+
+import static com.google.common.truth.Truth.assertWithMessage;
+
+import org.antlr.v4.runtime.CharStream;
+import org.antlr.v4.runtime.CharStreams;
+import org.junit.jupiter.api.Test;
+
+public class CrAwareLexerTest {
+
+    @Test
+    public void testConsumeCarriageReturnZeroCharPositionInLine() {
+        final String text = "\r";
+        final CharStream charStream = CharStreams.fromString(text);
+        final CrAwareLexerSimulator lexer =
+                new CrAwareLexerSimulator(null, null, null, null);
+        lexer.consume(charStream);
+
+        assertWithMessage("Carriage return should reset column number.")
+                .that(lexer.getCharPositionInLine())
+                .isEqualTo(0);
+    }
+
+    @Test
+    public void testConsumeCarriageReturnNewline() {
+        final String text = "\r";
+        final CharStream charStream = CharStreams.fromString(text);
+        final CrAwareLexerSimulator lexer =
+                new CrAwareLexerSimulator(null, null, null, null);
+        lexer.consume(charStream);
+
+        assertWithMessage("Carriage return should increment line number.")
+                .that(lexer.getLine())
+                .isEqualTo(2);
+    }
+
+    @Test
+    public void testConsumeWindowsNewlineZeroCharPositionInLine() {
+        final String text = "\r\n";
+        final CharStream charStream = CharStreams.fromString(text);
+        final CrAwareLexerSimulator lexer =
+                new CrAwareLexerSimulator(null, null, null, null);
+        lexer.consume(charStream);
+        lexer.consume(charStream);
+
+        assertWithMessage("Carriage return should reset column number.")
+                .that(lexer.getCharPositionInLine())
+                .isEqualTo(0);
+    }
+
+    @Test
+    public void testConsumeWindowsNewline() {
+        final String text = "\r\n";
+        final CharStream charStream = CharStreams.fromString(text);
+        final CrAwareLexerSimulator lexer =
+                new CrAwareLexerSimulator(null, null, null, null);
+        lexer.consume(charStream);
+        lexer.consume(charStream);
+
+        assertWithMessage("Carriage return should increment line number.")
+                .that(lexer.getLine())
+                .isEqualTo(2);
+    }
+}
diff --git a/src/test/java/com/puppycrawl/tools/checkstyle/grammar/javadoc/JavadocParseTreeTest.java b/src/test/java/com/puppycrawl/tools/checkstyle/grammar/javadoc/JavadocParseTreeTest.java
--- a/src/test/java/com/puppycrawl/tools/checkstyle/grammar/javadoc/JavadocParseTreeTest.java
+++ b/src/test/java/com/puppycrawl/tools/checkstyle/grammar/javadoc/JavadocParseTreeTest.java
@@ -394,4 +394,15 @@ public void testCdata() throws Exception {
                 getHtmlPath("InputCdata.javadoc"));
     }
 
+    @Test
+    public void testCrAsNewlineMultiline() throws Exception {
+        verifyJavadocTree(getPath("expectedCrAsNewlineMultiline.txt"),
+                getPath("InputCrAsNewlineMultiline.javadoc"));
+    }
+
+    @Test
+    public void testDosLineEndingAsNewlineMultiline() throws Exception {
+        verifyJavadocTree(getPath("expectedDosLineEndingAsNewlineMultiline.txt"),
+                getPath("InputDosLineEndingAsNewlineMultiline.javadoc"));
+    }
 }
diff --git a/src/test/resources/com/puppycrawl/tools/checkstyle/grammar/javadoc/InputCrAsNewlineMultiline.javadoc b/src/test/resources/com/puppycrawl/tools/checkstyle/grammar/javadoc/InputCrAsNewlineMultiline.javadoc
new file mode 100644
--- /dev/null
+++ b/src/test/resources/com/puppycrawl/tools/checkstyle/grammar/javadoc/InputCrAsNewlineMultiline.javadoc
@@ -0,0 +1 @@
+* line 0* Checks that commentContent starts with '*' javadoc comment identifier.    *&lt;property name="ignorePattern" value="^ *\* *[^ ]+$"/&gt** This * is not a leading ** @return true if commentContent starts with '*' javadoc comment identifier* line 5    * line 6
\ No newline at end of file
diff --git a/src/test/resources/com/puppycrawl/tools/checkstyle/grammar/javadoc/InputDosLineEndingAsNewlineMultiline.javadoc b/src/test/resources/com/puppycrawl/tools/checkstyle/grammar/javadoc/InputDosLineEndingAsNewlineMultiline.javadoc
new file mode 100644
--- /dev/null
+++ b/src/test/resources/com/puppycrawl/tools/checkstyle/grammar/javadoc/InputDosLineEndingAsNewlineMultiline.javadoc
@@ -0,0 +1,7 @@
+* line 0
+* Checks that commentContent starts with '*' javadoc comment identifier.
+    *&lt;property name="ignorePattern" value="^ *\* *[^ ]+$"/&gt*
+* This * is not a leading *
+* @return true if commentContent starts with '*' javadoc comment identifier
+* line 5
+    * line 6
diff --git a/src/test/resources/com/puppycrawl/tools/checkstyle/grammar/javadoc/expectedCrAsNewlineMultiline.txt b/src/test/resources/com/puppycrawl/tools/checkstyle/grammar/javadoc/expectedCrAsNewlineMultiline.txt
new file mode 100644
--- /dev/null
+++ b/src/test/resources/com/puppycrawl/tools/checkstyle/grammar/javadoc/expectedCrAsNewlineMultiline.txt
@@ -0,0 +1,28 @@
+JAVADOC -> JAVADOC [0:0]
+|--LEADING_ASTERISK -> * [0:0]
+|--TEXT ->  line 0 [0:1]
+|--NEWLINE -> \r [0:8]
+|--LEADING_ASTERISK -> * [1:0]
+|--TEXT ->  Checks that commentContent starts with '*' javadoc comment identifier. [1:1]
+|--NEWLINE -> \r [1:72]
+|--LEADING_ASTERISK ->     * [2:0]
+|--TEXT -> &lt;property name="ignorePattern" value="^ *\* *[^ ]+$"/&gt* [2:5]
+|--NEWLINE -> \r [2:65]
+|--LEADING_ASTERISK -> * [3:0]
+|--TEXT ->  This * is not a leading * [3:1]
+|--NEWLINE -> \r [3:27]
+|--LEADING_ASTERISK -> * [4:0]
+|--WS ->   [4:1]
+|--JAVADOC_TAG -> JAVADOC_TAG [4:2]
+|   |--RETURN_LITERAL -> @return [4:2]
+|   |--WS ->   [4:9]
+|   `--DESCRIPTION -> DESCRIPTION [4:10]
+|       |--TEXT -> true if commentContent starts with '*' javadoc comment identifier [4:10]
+|       |--NEWLINE -> \r [4:75]
+|       |--LEADING_ASTERISK -> * [5:0]
+|       |--TEXT ->  line 5 [5:1]
+|       |--NEWLINE -> \r [5:8]
+|       |--LEADING_ASTERISK ->     * [6:0]
+|       |--TEXT ->  line 6 [6:5]
+|       `--NEWLINE -> \r [6:12]
+`--EOF -> <EOF> [7:0]
diff --git a/src/test/resources/com/puppycrawl/tools/checkstyle/grammar/javadoc/expectedDosLineEndingAsNewlineMultiline.txt b/src/test/resources/com/puppycrawl/tools/checkstyle/grammar/javadoc/expectedDosLineEndingAsNewlineMultiline.txt
new file mode 100644
--- /dev/null
+++ b/src/test/resources/com/puppycrawl/tools/checkstyle/grammar/javadoc/expectedDosLineEndingAsNewlineMultiline.txt
@@ -0,0 +1,28 @@
+JAVADOC -> JAVADOC [0:0]
+|--LEADING_ASTERISK -> * [0:0]
+|--TEXT ->  line 0 [0:1]
+|--NEWLINE -> \n [0:8]
+|--LEADING_ASTERISK -> * [1:0]
+|--TEXT ->  Checks that commentContent starts with '*' javadoc comment identifier. [1:1]
+|--NEWLINE -> \n [1:72]
+|--LEADING_ASTERISK ->     * [2:0]
+|--TEXT -> &lt;property name="ignorePattern" value="^ *\* *[^ ]+$"/&gt* [2:5]
+|--NEWLINE -> \n [2:65]
+|--LEADING_ASTERISK -> * [3:0]
+|--TEXT ->  This * is not a leading * [3:1]
+|--NEWLINE -> \n [3:27]
+|--LEADING_ASTERISK -> * [4:0]
+|--WS ->   [4:1]
+|--JAVADOC_TAG -> JAVADOC_TAG [4:2]
+|   |--RETURN_LITERAL -> @return [4:2]
+|   |--WS ->   [4:9]
+|   `--DESCRIPTION -> DESCRIPTION [4:10]
+|       |--TEXT -> true if commentContent starts with '*' javadoc comment identifier [4:10]
+|       |--NEWLINE -> \n [4:75]
+|       |--LEADING_ASTERISK -> * [5:0]
+|       |--TEXT ->  line 5 [5:1]
+|       |--NEWLINE -> \n [5:8]
+|       |--LEADING_ASTERISK ->     * [6:0]
+|       |--TEXT ->  line 6 [6:5]
+|       `--NEWLINE -> \n [6:12]
+`--EOF -> <EOF> [7:0]
diff --git a/src/test/resources/com/puppycrawl/tools/checkstyle/grammar/javadoc/expectedJavadocWithCrAsNewlineAst.txt b/src/test/resources/com/puppycrawl/tools/checkstyle/grammar/javadoc/expectedJavadocWithCrAsNewlineAst.txt
--- a/src/test/resources/com/puppycrawl/tools/checkstyle/grammar/javadoc/expectedJavadocWithCrAsNewlineAst.txt
+++ b/src/test/resources/com/puppycrawl/tools/checkstyle/grammar/javadoc/expectedJavadocWithCrAsNewlineAst.txt
@@ -1,20 +1,20 @@
 JAVADOC -> JAVADOC [0:0]
 |--NEWLINE -> \r [0:0]
-|--LEADING_ASTERISK -> * [0:1]
-|--TEXT ->  Checks that commentContent starts with '*' javadoc comment identifier. [0:2]
-|--NEWLINE -> \r [0:73]
-|--LEADING_ASTERISK ->     * [0:74]
-|--TEXT -> &lt;property name="ignorePattern" value="^ *\* *[^ ]+$"/&gt* [0:79]
-|--NEWLINE -> \r [0:139]
-|--LEADING_ASTERISK -> \t* [0:140]
-|--TEXT ->  This * is not a leading * [0:142]
-|--NEWLINE -> \r [0:168]
-|--LEADING_ASTERISK -> * [0:169]
-|--WS ->   [0:170]
-|--JAVADOC_TAG -> JAVADOC_TAG [0:171]
-|   |--RETURN_LITERAL -> @return [0:171]
-|   |--WS ->   [0:178]
-|   `--DESCRIPTION -> DESCRIPTION [0:179]
-|       |--TEXT -> true if commentContent starts with '*' javadoc comment identifier [0:179]
-|       `--NEWLINE -> \r [0:244]
-`--EOF -> <EOF> [0:245]
+|--LEADING_ASTERISK -> * [1:0]
+|--TEXT ->  Checks that commentContent starts with '*' javadoc comment identifier. [1:1]
+|--NEWLINE -> \r [1:72]
+|--LEADING_ASTERISK ->     * [2:0]
+|--TEXT -> &lt;property name="ignorePattern" value="^ *\* *[^ ]+$"/&gt* [2:5]
+|--NEWLINE -> \r [2:65]
+|--LEADING_ASTERISK -> \t* [3:0]
+|--TEXT ->  This * is not a leading * [3:2]
+|--NEWLINE -> \r [3:28]
+|--LEADING_ASTERISK -> * [4:0]
+|--WS ->   [4:1]
+|--JAVADOC_TAG -> JAVADOC_TAG [4:2]
+|   |--RETURN_LITERAL -> @return [4:2]
+|   |--WS ->   [4:9]
+|   `--DESCRIPTION -> DESCRIPTION [4:10]
+|       |--TEXT -> true if commentContent starts with '*' javadoc comment identifier [4:10]
+|       `--NEWLINE -> \r [4:75]
+`--EOF -> <EOF> [5:0]