Repository: redpen-cc/redpen
Pull request #601 (2016-04-06 07:42:40)

[WIP] Re:VIEW parser
Ref: #590 


Patch:
diff --git a/redpen-core/src/main/java/cc/redpen/parser/DocumentParser.java b/redpen-core/src/main/java/cc/redpen/parser/DocumentParser.java
index 78d0108a..bac3d880 100644
--- a/redpen-core/src/main/java/cc/redpen/parser/DocumentParser.java
+++ b/redpen-core/src/main/java/cc/redpen/parser/DocumentParser.java
@@ -20,6 +20,7 @@ package cc.redpen.parser;
 import cc.redpen.RedPenException;
 import cc.redpen.model.Document;
 import cc.redpen.parser.asciidoc.AsciiDocParser;
+import cc.redpen.parser.review.ReVIEWParser;
 import cc.redpen.tokenizer.RedPenTokenizer;
 
 import java.io.File;
@@ -90,6 +91,7 @@ public interface DocumentParser {
     DocumentParser LATEX = new LaTeXParser();
     DocumentParser ASCIIDOC = new AsciiDocParser();
     DocumentParser PROPERTIES = new PropertiesParser();
+    DocumentParser REVIEW = new ReVIEWParser();
 
     Map<String, DocumentParser> PARSER_MAP = Collections.unmodifiableMap(
             new HashMap<String, DocumentParser>() {
@@ -100,6 +102,7 @@ public interface DocumentParser {
                     put("LATEX", LATEX);
                     put("ASCIIDOC", ASCIIDOC);
                     put("PROPERTIES", PROPERTIES);
+                    put("REVIEW", REVIEW);
                 }
             });
 
diff --git a/redpen-core/src/main/java/cc/redpen/parser/asciidoc/AsciiDocLine.java b/redpen-core/src/main/java/cc/redpen/parser/asciidoc/AsciiDocLine.java
new file mode 100644
index 00000000..e5253598
--- /dev/null
+++ b/redpen-core/src/main/java/cc/redpen/parser/asciidoc/AsciiDocLine.java
@@ -0,0 +1,75 @@
+/**
+ * redpen: a text inspection tool
+ * Copyright (c) 2014-2015 Recruit Technologies Co., Ltd. and contributors
+ * (see CONTRIBUTORS.md)
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cc.redpen.parser.asciidoc;
+
+import cc.redpen.parser.common.Line;
+
+
+/**
+ * An 'erasing' string utility class that stores the original offset for each preserved character
+ */
+public class AsciiDocLine extends Line {
+    /**
+     * Construct a line using the supplied string
+     *
+     * @param str    the text of the line
+     * @param lineno the original line number
+     */
+    public AsciiDocLine(String str, int lineno) {
+        super(str, lineno);
+        this.lineNo = lineno;
+        this.inlineMarkupDelimiters = " _*`#^~.,";
+        if (!str.isEmpty()) {
+            allSameCharacter = true;
+            char lastCh = 0;
+            for (int i = 0; i < str.length(); i++) {
+                char ch = str.charAt(i);
+
+                if ((i < str.length() - 1) && (ch == '\\')) {
+                    i++;
+                    ch = str.charAt(i);
+                    escaped.add(true);
+                }
+                else {
+                    escaped.add(false);
+                }
+
+                offsets.add(i);
+                characters.add(ch);
+                valid.add(true);
+
+                if ((lastCh != 0) && (lastCh != ch)) {
+                    allSameCharacter = false;
+                }
+                lastCh = ch;
+            }
+        }
+
+        // trim the end
+        while (!characters.isEmpty() &&
+                Character.isWhitespace(characters.get(characters.size() - 1))) {
+            characters.remove(characters.size() - 1);
+        }
+    }
+
+    @Override
+    public String toString() {
+        return super.toString();
+    }
+}
diff --git a/redpen-core/src/main/java/cc/redpen/parser/asciidoc/AsciiDocParser.java b/redpen-core/src/main/java/cc/redpen/parser/asciidoc/AsciiDocParser.java
index 07cfab14..26bdc9bd 100644
--- a/redpen-core/src/main/java/cc/redpen/parser/asciidoc/AsciiDocParser.java
+++ b/redpen-core/src/main/java/cc/redpen/parser/asciidoc/AsciiDocParser.java
@@ -18,19 +18,14 @@
 
 package cc.redpen.parser.asciidoc;
 
-import cc.redpen.RedPenException;
-import cc.redpen.model.Document;
-import cc.redpen.parser.BaseDocumentParser;
-import cc.redpen.parser.SentenceExtractor;
-import cc.redpen.tokenizer.RedPenTokenizer;
+import cc.redpen.parser.common.Line;
+import cc.redpen.parser.common.LineParser;
+import cc.redpen.parser.common.Model;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 import java.io.BufferedReader;
 import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Optional;
 
 
 /**
@@ -47,7 +42,7 @@ import java.util.Optional;
  * <p>
  * AsciiDoc's syntax and grammar is documented at @see http://asciidoc.org/
  */
-public class AsciiDocParser extends BaseDocumentParser {
+public class AsciiDocParser extends LineParser {
 
     private static final Logger LOG = LoggerFactory.getLogger(AsciiDocParser.class);
 
@@ -100,29 +95,12 @@ public class AsciiDocParser extends BaseDocumentParser {
     }
 
 
-    @Override
-    public Document parse(InputStream inputStream, Optional<String> fileName, SentenceExtractor sentenceExtractor, RedPenTokenizer tokenizer) throws RedPenException {
-        Document.DocumentBuilder documentBuilder = Document.builder(tokenizer);
-        fileName.ifPresent(documentBuilder::setFileName);
-
-        Model model = new Model(sentenceExtractor);
-
-        // add the lines from the input to the model
-        populateModel(model, inputStream);
-
-        // convert the model to a redpen document
-        convertModel(model, documentBuilder);
-
-        return documentBuilder.build();
-    }
-
     /**
      * populate the erasable model with the text from the inputstream
      * @param model model to populate
      * @param io stream to read
      */
     protected void populateModel(Model model, InputStream io) {
-        State state = new State();
         BufferedReader reader = createReader(io);
 
         int lineno = 0;
@@ -134,11 +112,12 @@ public class AsciiDocParser extends BaseDocumentParser {
                     break;
                 }
                 lineno++;
-                model.add(new Line(line, lineno));
+                model.add(new AsciiDocLine(line, lineno));
             }
             reader.close();
 
-            // preocess each line of the model
+            // process each line of the model
+            State state = new State();
             for (model.rewind(); model.isMore(); model.getNextLine()) {
                 processLine(model.getCurrentLine(), model, state);
             }
@@ -154,69 +133,6 @@ public class AsciiDocParser extends BaseDocumentParser {
         }
     }
 
-    /**
-     * Conver the parser's model to the RedPen document model
-     *
-     * @param model model to convert
-     * @param builder doc builder
-     */
-    protected void convertModel(Model model, Document.DocumentBuilder builder) {
-        model.rewind();
-
-        // add a header if there isn't one in the model
-        if ((model.getCurrentLine() != null) && (model.getCurrentLine().getSectionLevel() == 0)) {
-            builder.addSection(0);
-        }
-
-        while (model.isMore()) {
-
-            // skip blank lines
-            while (model.isMore() && model.getCurrentLine().isEmpty()) {
-                model.getNextLine();
-            }
-
-            if (model.isMore()) {
-                // check for new sections
-                if (model.getCurrentLine().getSectionLevel() > 0) {
-                    builder.addSection(
-                            model.getCurrentLine().getSectionLevel(),
-                            model.convertToSentences(model.getCurrentLine())
-                    );
-                    model.getNextLine();
-                }
-                // check for a list item
-                else if (model.getCurrentLine().isListStart()) {
-                    List<Line> listElementLines = new ArrayList<>();
-                    int listLevel = model.getCurrentLine().getListLevel();
-
-                    // add the list start line
-                    listElementLines.add(model.getCurrentLine());
-
-                    // test the following lines to see if they continue this list item
-                    model.getNextLine();
-                    while (model.isMore() &&
-                            !model.getCurrentLine().isListStart() &&
-                            (model.getCurrentLine().getListLevel() == listLevel)) {
-                        listElementLines.add(model.getCurrentLine());
-                        model.getNextLine();
-                    }
-                    builder.addListElement(listLevel, model.convertToSentences(listElementLines));
-                }
-                // process a paragraph
-                else {
-                    List<Line> paragraphLines = new ArrayList<>();
-                    // current line can't be empty, so this loop will enter at least once
-                    while (model.isMore() && !model.getCurrentLine().isEmpty()) {
-                        paragraphLines.add(model.getCurrentLine());
-                        model.getNextLine();
-                    }
-                    builder.addParagraph();
-                    model.convertToSentences(paragraphLines).forEach(builder::addSentence);
-                }
-            }
-        }
-    }
-
     /**
      * Does the give line start a list?
      *
@@ -314,214 +230,204 @@ public class AsciiDocParser extends BaseDocumentParser {
      * @param state the current state
      */
     private void processLine(Line line, Model model, State state) {
+        if (line.isErased()) { return; }
 
-        if (!line.isErased()) {
-
-            Line previousLine = model.getLine(line.getLineNo() - 1);
-            Line nextLine = model.getLine(line.getLineNo() + 1);
+        TargetLine target = new TargetLine(line,
+                model.getLine(line.getLineNo() - 1),
+                model.getLine(line.getLineNo() + 1));
 
-            if (state.inList && (line.getListLevel() == 0)) {
-                line.setListLevel(previousLine.getListLevel());
-            }
-
-            char firstChar = line.charAt(0);
-            char secondChar = line.charAt(1);
-
-            // check for block end
-            if (state.inBlock) {
-                if (line.isAllSameCharacter() &&
-                        (firstChar == state.blockMarker) &&
-                        (line.length() == state.blockMarkerLength)) {
-                    // end a regular block
-                    line.erase();
-                    line.setInBlock(true);
-                    state.inBlock = false;
-                    return;
-                }
-                else if ((line.length() >= 4) &&
-                        (firstChar == state.blockMarker) &&
-                        (firstChar == '|') &&
-                        (secondChar == '=')) {
-                    // end a table
-                    line.erase();
-                    line.setInBlock(true);
-                    state.inBlock = false;
-                    return;
-                }
-                // erase the block content
-                line.setInBlock(true);
-                if (state.eraseBlock) {
-                    line.erase();
-                    return;
-                }
-            }
+        if (state.inList && (line.getListLevel() == 0)) {
+            line.setListLevel(target.previousLine.getListLevel());
+        }
 
-            // check for old style heading (line followed by single-char-line of same length)
+        // check for block end
+        if (state.inBlock) {
             if (line.isAllSameCharacter() &&
-                    (line.length() == previousLine.length()) &&
-                    ("=-~^+".indexOf(firstChar) != -1) &&
-                    (". [".indexOf(previousLine.charAt(0, true)) == -1)) {
-                previousLine.setSectionLevel(1);
+                    (target.firstChar == state.blockMarker) &&
+                    (line.length() == state.blockMarkerLength)) {
+                // end a regular block
                 line.erase();
+                line.setInBlock(true);
+                state.inBlock = false;
                 return;
-            }
-
-            // horizontal rule
-            if (line.isAllSameCharacter() && (line.length() == 3) && (line.charAt(0) == '\'')) {
+            } else if ((line.length() >= 4) &&
+                    (target.firstChar == state.blockMarker) &&
+                    (target.firstChar == '|') &&
+                    (target.secondChar == '=')) {
+                // end a table
                 line.erase();
+                line.setInBlock(true);
+                state.inBlock = false;
                 return;
             }
-
-            // block markers in which we process the internal text
-            if (line.isAllSameCharacter() && (line.length() >= 4) &&
-                    ("_*".indexOf(firstChar) != -1)) {
+            // erase the block content
+            line.setInBlock(true);
+            if (state.eraseBlock) {
                 line.erase();
                 return;
             }
+        }
 
-            // test for various block starts
-            if (!state.inBlock) {
-
-                // fenced block
-                if (line.isAllSameCharacter() && (line.length() == 3) && (line.charAt(0) == '`')) {
-                    state.inBlock = true;
-                    state.eraseBlock = true;
-                    state.blockMarker = firstChar;
-                    state.blockMarkerLength = line.length();
-                    line.setInBlock(true);
-                    line.erase();
-                    return;
-                }
-
-                // see if we are starting other types of blocks
-                if (line.isAllSameCharacter() && (line.length() >= 4)) {
-                    switch (firstChar) {
-                        case '-':
-                        case '=':
-                        case '&':
-                        case '/':
-                        case '+':
-                        case '.':
-                            // blocks that have their innards erased
-                            state.inBlock = true;
-                            state.eraseBlock = true;
-                            state.blockMarker = firstChar;
-                            state.blockMarkerLength = line.length();
-                            line.setInBlock(true);
-                            line.erase();
-                            return;
-                    }
-                }
+        // check for old style heading (line followed by single-char-line of same length)
+        if (line.isAllSameCharacter() &&
+                (line.length() == target.previousLine.length()) &&
+                ("=-~^+".indexOf(target.firstChar) != -1) &&
+                (". [".indexOf(target.previousLine.charAt(0, true)) == -1)) {
+            target.previousLine.setSectionLevel(1);
+            line.erase();
+            return;
+        }
 
-                // see if this is a table marker
-                if ((line.length() >= 4) && (firstChar == '|') && (secondChar == '=')) {
-                    line.erase();
-                    state.inBlock = true;
-                    state.eraseBlock = true;
-                    state.blockMarker = '|';
-                    state.blockMarkerLength = 1;
-                    line.setInBlock(true);
-                    return;
-                }
+        // horizontal rule
+        if (line.isAllSameCharacter() && (line.length() == 3) && (line.charAt(0) == '\'')) {
+            line.erase();
+            return;
+        }
 
-            }
+        // block markers in which we process the internal text
+        if (line.isAllSameCharacter() && (line.length() >= 4) &&
+                ("_*".indexOf(target.firstChar) != -1)) {
+            line.erase();
+            return;
+        }
 
-            // check for a single-line literal sentence
-            if (!state.inList && (firstChar == ' ')) {
+        // test for various block starts
+        if (!state.inBlock) {
+            // fenced block
+            if (line.isAllSameCharacter() && (line.length() == 3) && (line.charAt(0) == '`')) {
+                state.inBlock = true;
+                state.eraseBlock = true;
+                state.blockMarker = target.firstChar;
+                state.blockMarkerLength = line.length();
+                line.setInBlock(true);
                 line.erase();
                 return;
             }
 
-            // maybe we have a comment?
-            if ((firstChar == '/') && (secondChar == '/')) {
-                line.erase();
-                return;
+            // see if we are starting other types of blocks
+            if (line.isAllSameCharacter() && (line.length() >= 4)) {
+                switch (target.firstChar) {
+                    case '-':
+                    case '=':
+                    case '&':
+                    case '/':
+                    case '+':
+                    case '.':
+                        // blocks that have their innards erased
+                        state.inBlock = true;
+                        state.eraseBlock = true;
+                        state.blockMarker = target.firstChar;
+                        state.blockMarkerLength = line.length();
+                        line.setInBlock(true);
+                        line.erase();
+                        return;
+                }
             }
 
-            // 'open' block marker
-            if ((firstChar == '-') && (secondChar == '-')) {
+            // see if this is a table marker
+            if ((line.length() >= 4) && (target.firstChar == '|') && (target.secondChar == '=')) {
                 line.erase();
+                state.inBlock = true;
+                state.eraseBlock = true;
+                state.blockMarker = '|';
+                state.blockMarkerLength = 1;
+                line.setInBlock(true);
                 return;
             }
 
-            // attributes (at position == 0)
-            if (line.eraseEnclosure(":", ":", Line.EraseStyle.None) == 0) {
-                line.erase();
-                return;
-            }
-            if (line.eraseEnclosure("[", "]", Line.EraseStyle.None) == 0) {
-                line.erase();
-                return;
-            }
+        }
 
-            // check for a title
-            if ((firstChar == '.') && (" .".indexOf(secondChar) == -1)) {
-                line.erase(0, 1);
-            }
+        // check for a single-line literal sentence
+        if (!state.inList && (target.firstChar == ' ')) {
+            line.erase();
+            return;
+        }
 
-            // erase urls and links
-            for (String prefix : EXTERNAL_LINK_PREFIXES) {
-                line.eraseEnclosure(prefix, " ,[", Line.EraseStyle.CloseMarkerContainsDelimiters);
-            }
+        // maybe we have a comment?
+        if ((target.firstChar == '/') && (target.secondChar == '/')) {
+            line.erase();
+            return;
+        }
+
+        // 'open' block marker
+        if ((target.firstChar == '-') && (target.secondChar == '-')) {
+            line.erase();
+            return;
+        }
+
+        // attributes (at position == 0)
+        if (line.eraseEnclosure(":", ":", AsciiDocLine.EraseStyle.None) == 0) {
+            line.erase();
+            return;
+        }
+        if (line.eraseEnclosure("[", "]", AsciiDocLine.EraseStyle.None) == 0) {
+            line.erase();
+            return;
+        }
 
-            // enclosed directives
-            line.eraseEnclosure("+++", "+++", Line.EraseStyle.All);
-            line.eraseEnclosure("[[", "]]", Line.EraseStyle.All);
+        // check for a title
+        if ((target.firstChar == '.') && (" .".indexOf(target.secondChar) == -1)) {
+            line.erase(0, 1);
+        }
 
-            line.eraseEnclosure("<<", ">>", Line.EraseStyle.PreserveLabel);
+        // erase urls and links
+        for (String prefix : EXTERNAL_LINK_PREFIXES) {
+            line.eraseEnclosure(prefix, " ,[", AsciiDocLine.EraseStyle.CloseMarkerContainsDelimiters);
+        }
 
-            line.eraseEnclosure("{", "}", Line.EraseStyle.Markers); // NOTE: should we make substitutions?
-            line.eraseEnclosure("[", "]", Line.EraseStyle.Markers);
+        // enclosed directives
+        line.eraseEnclosure("+++", "+++", AsciiDocLine.EraseStyle.All);
+        line.eraseEnclosure("[[", "]]", AsciiDocLine.EraseStyle.All);
+        line.eraseEnclosure("<<", ">>", AsciiDocLine.EraseStyle.PreserveLabel);
+        line.eraseEnclosure("{", "}", AsciiDocLine.EraseStyle.Markers); // NOTE: should we make substitutions?
+        line.eraseEnclosure("[", "]", AsciiDocLine.EraseStyle.Markers);
 
-            // headers
-            int headerIndent = 0;
-            while (line.charAt(headerIndent) == '=') {
-                headerIndent++;
-            }
-            if ((headerIndent > 0) && (line.charAt(headerIndent) == ' ')) {
-                line.erase(0, headerIndent + 1);
-                line.setSectionLevel(headerIndent);
-            }
+        int headerIndent = 0;
+        while (line.charAt(headerIndent) == '=') {
+            headerIndent++;
+        }
+        if ((headerIndent > 0) && (line.charAt(headerIndent) == ' ')) {
+            line.erase(0, headerIndent + 1);
+            line.setSectionLevel(headerIndent);
+        }
 
-            // lists!
-            if (!state.inBlock && isListElement(line, nextLine)) {
-                state.inList = true;
-            }
+        // lists!
+        if (!state.inBlock && isListElement(line, target.nextLine)) {
+            state.inList = true;
+        }
 
-            // continuation markers
-            if (line.charAt(line.length() - 1) == '+') {
-                if (line.length() == 0) {
-                    line.erase();
-                }
-                else {
-                    line.erase(line.length() - 1, 1);
-                }
+        // continuation markers
+        if (line.charAt(line.length() - 1) == '+') {
+            if (line.length() == 0) {
+                line.erase();
+            } else {
+                line.erase(line.length() - 1, 1);
             }
+        }
 
-            // a blank line will cancel any list element we are in
-            if (state.inList && (line.length() == 0)) {
-                state.inList = false;
-                line.setListLevel(0);
-            }
+        // a blank line will cancel any list element we are in
+        if (state.inList && (line.length() == 0)) {
+            state.inList = false;
+            line.setListLevel(0);
+        }
 
-            // macros
-            for (String macro : MACROS) {
-                if (line.startsWith(macro)) {
-                    line.erase();
-                    break;
-                }
+        // macros
+        for (String macro : MACROS) {
+            if (line.startsWith(macro)) {
+                line.erase();
+                break;
             }
+        }
 
-            // admonitions
-            for (String admonition : ADMONITIONS) {
-                if (line.startsWith(admonition)) {
-                    line.erase(0, admonition.length());
-                    break;
-                }
+        // admonitions
+        for (String admonition : ADMONITIONS) {
+            if (line.startsWith(admonition)) {
+                line.erase(0, admonition.length());
+                break;
             }
-
-            eraseInlineMarkup(line);
         }
+
+        eraseInlineMarkup(line);
     }
 
 
@@ -532,17 +438,17 @@ public class AsciiDocParser extends BaseDocumentParser {
      */
     private void eraseInlineMarkup(Line line) {
         // inline markup (bold, italics etc)
-        line.eraseEnclosure("__", "__", Line.EraseStyle.Markers);
-        line.eraseEnclosure("**", "**", Line.EraseStyle.Markers);
-        line.eraseEnclosure("``", "``", Line.EraseStyle.Markers);
-        line.eraseEnclosure("##", "##", Line.EraseStyle.Markers);
-        line.eraseEnclosure("^", "^", Line.EraseStyle.Markers);
-        line.eraseEnclosure("~", "~", Line.EraseStyle.Markers);
-
-        line.eraseEnclosure("_", "_", Line.EraseStyle.InlineMarkup);
-        line.eraseEnclosure("*", "*", Line.EraseStyle.InlineMarkup);
-        line.eraseEnclosure("`", "`", Line.EraseStyle.InlineMarkup);
-        line.eraseEnclosure("#", "#", Line.EraseStyle.InlineMarkup);
+        line.eraseEnclosure("__", "__", AsciiDocLine.EraseStyle.Markers);
+        line.eraseEnclosure("**", "**", AsciiDocLine.EraseStyle.Markers);
+        line.eraseEnclosure("``", "``", AsciiDocLine.EraseStyle.Markers);
+        line.eraseEnclosure("##", "##", AsciiDocLine.EraseStyle.Markers);
+        line.eraseEnclosure("^", "^", AsciiDocLine.EraseStyle.Markers);
+        line.eraseEnclosure("~", "~", AsciiDocLine.EraseStyle.Markers);
+
+        line.eraseEnclosure("_", "_", AsciiDocLine.EraseStyle.InlineMarkup);
+        line.eraseEnclosure("*", "*", AsciiDocLine.EraseStyle.InlineMarkup);
+        line.eraseEnclosure("`", "`", AsciiDocLine.EraseStyle.InlineMarkup);
+        line.eraseEnclosure("#", "#", AsciiDocLine.EraseStyle.InlineMarkup);
 
         line.erase("'`");
         line.erase("`'");
@@ -553,4 +459,3 @@ public class AsciiDocParser extends BaseDocumentParser {
         line.erase("(TM)");
     }
 }
-
diff --git a/redpen-core/src/main/java/cc/redpen/parser/asciidoc/Line.java b/redpen-core/src/main/java/cc/redpen/parser/common/Line.java
similarity index 60%
rename from redpen-core/src/main/java/cc/redpen/parser/asciidoc/Line.java
rename to redpen-core/src/main/java/cc/redpen/parser/common/Line.java
index 0f0fa579..5a667333 100644
--- a/redpen-core/src/main/java/cc/redpen/parser/asciidoc/Line.java
+++ b/redpen-core/src/main/java/cc/redpen/parser/common/Line.java
@@ -16,19 +16,35 @@
  * limitations under the License.
  */
 
-package cc.redpen.parser.asciidoc;
+package cc.redpen.parser.common;
 
 import java.util.ArrayList;
 import java.util.List;
 
-
-/**
- * An 'erasing' string utility class that stores the original offset for each preserved character
- */
 public class Line {
     // value returned for comparison if a character is escaped
     static final char ESCAPED_CHARACTER_VALUE = 'ø';
-    static final String INLINE_MARKUP_DELIMITERS = " _*`#^~.,";
+
+    // a list of offsets for each character
+    protected List<Integer> offsets = new ArrayList<>();
+    // the character list for the line
+    protected List<Character> characters = new ArrayList<>();
+    // the text of the line
+    protected String text;
+    // marks erased characters as invalid
+    protected List<Boolean> valid = new ArrayList<>();
+    // remembers which characters were escaped in the original string
+    protected List<Boolean> escaped = new ArrayList<>();
+    // Whole line is erased.
+    protected boolean erased = false;
+
+    protected int lineNo = 0;
+    protected boolean allSameCharacter = false;
+    protected boolean inBlock = false;
+    protected int sectionLevel = 0;
+    protected int listLevel = 0;
+    protected boolean listStart = false;
+    protected String inlineMarkupDelimiters = " ";
 
     /**
      * The different ways embedded inline markers can be erased
@@ -42,135 +58,9 @@ public class Line {
         CloseMarkerContainsDelimiters
     }
 
-    // a list of offsets for each character
-    List<Integer> offsets = new ArrayList<>();
-    // the text for the line
-    List<Character> text = new ArrayList<>();
-    // marks erased characters as invalid
-    List<Boolean> valid = new ArrayList<>();
-    // remembers which characters were escaped in the original string
-    List<Boolean> escaped = new ArrayList<>();
-
-    private int lineNo = 0;
-    private boolean allSameCharacter = false;
-    private boolean erased = false;
-    private boolean inBlock = false;
-
-    private int sectionLevel = 0;
-    private int listLevel = 0;
-    private boolean listStart = false;
-
-    /**
-     * Construct a line using the supplied string
-     *
-     * @param str    the text of the line
-     * @param lineno the original line number
-     */
-    public Line(String str, int lineno) {
-        this.lineNo = lineno;
-        if (!str.isEmpty()) {
-            allSameCharacter = true;
-            char lastCh = 0;
-            for (int i = 0; i < str.length(); i++) {
-                char ch = str.charAt(i);
-
-                if ((i < str.length() - 1) && (ch == '\\')) {
-                    i++;
-                    ch = str.charAt(i);
-                    escaped.add(true);
-                }
-                else {
-                    escaped.add(false);
-                }
-
-                offsets.add(i);
-                text.add(ch);
-                valid.add(true);
-
-                if ((lastCh != 0) && (lastCh != ch)) {
-                    allSameCharacter = false;
-                }
-                lastCh = ch;
-            }
-        }
-
-        // trim the end
-        while (!text.isEmpty() &&
-                Character.isWhitespace(text.get(text.size() - 1))) {
-            text.remove(text.size() - 1);
-        }
-    }
-
-    /**
-     * Is this line a repeating set of the same character?
-     *
-     * @return true is all same character
-     */
-    public boolean isAllSameCharacter() {
-        return allSameCharacter;
-    }
-
-    /**
-     * Has this line been completely erased?
-     *
-     * @return true if erased
-     */
-    public boolean isErased() {
-        return erased;
-    }
-
-    /**
-     * Is this line in a block?
-     *
-     * @return true is it's in a block
-     */
-    public boolean isInBlock() {
-        return inBlock;
-    }
-
-
-    /**
-     * Is this line the start of a new list item?
-     *
-     * @return true if it's list start
-     */
-    public boolean isListStart() {
-        return listStart;
-    }
-
-    /**
-     * Return the list level for this line, or zero if it is not in a list.
-     *
-     * @return the list level, or zero if not in a list
-     */
-    public int getListLevel() {
-        return listLevel;
-    }
-
-    /**
-     * If the line is a section header, return the section level,
-     * or zero if the line is not a section header
-     *
-     * @return the section level or zero if not a section header
-     */
-    public int getSectionLevel() {
-        return sectionLevel;
-    }
-
-    public void setInBlock(boolean inBlock) {
-        this.inBlock = inBlock;
-    }
-
-    public void setSectionLevel(int newSectionLevel) {
-        this.sectionLevel = newSectionLevel;
-    }
-
-    public void setListLevel(int newListLevel) {
-        this.listLevel = newListLevel;
-    }
-
-    public void setListStart(boolean listStart) {
-        this.listStart = listStart;
+    public Line(String str, int lineNo) {
+        this.lineNo = lineNo;
+        this.text = str;
     }
 
     /**
@@ -203,7 +93,7 @@ public class Line {
      * @param segment segment to be erased
      */
     public void erase(String segment) {
-        for (int i = 0; i < text.size(); i++) {
+        for (int i = 0; i < characters.size(); i++) {
             boolean found = true;
             for (int j = 0; j < segment.length(); j++) {
                 if (charAt(j + i) != segment.charAt(j)) {
@@ -218,115 +108,13 @@ public class Line {
         }
     }
 
-    /**
-     * Erase the open and close markers, and optionally all the text inside them
-     * Returns the position of the first enclosure or -1 if no enclosure was found
-     *
-     * @param open open marker to be erased
-     * @param close close marker to be erased
-     * @param eraseStyle erase style
-     * @return position of first enclosure
-     */
-    public int eraseEnclosure(String open,
-                              String close,
-                              EraseStyle eraseStyle) {
-        boolean inEnclosure = false;
-        int firstEnclosurePosition = -1;
-        int lastCommaPosition = -1;
-        int enclosureStart = 0;
-        for (int i = 0; i < length(); i++) {
-            if (valid.get(i)) {
-                if (!inEnclosure) {
-                    // look for the open string
-                    boolean foundOpen = true;
-                    for (int j = 0; j < open.length(); j++) {
-                        if (charAt(i + j) != open.charAt(j)) {
-                            foundOpen = false;
-                            break;
-                        }
-                    }
-                    // inline requires start of line or a space before the marker
-                    if (foundOpen && (eraseStyle == EraseStyle.InlineMarkup)) {
-                        if ((i != 0) &&
-                                (INLINE_MARKUP_DELIMITERS.indexOf(charAt(i - 1)) == -1)) {
-                            foundOpen = false;
-                        }
-                    }
-                    if (foundOpen) {
-                        enclosureStart = i;
-                        inEnclosure = true;
-                        firstEnclosurePosition = i;
-                    }
-                }
-                else {
-                    // look for the close string
-                    boolean foundClose = true;
-                    if (eraseStyle == EraseStyle.CloseMarkerContainsDelimiters) {
-                        foundClose = (i == length() - 1) || (close.indexOf(charAt(i)) != -1);
-                    }
-                    else {
-                        for (int j = 0; j < close.length(); j++) {
-                            if (charAt(i + j) != close.charAt(j)) {
-                                foundClose = false;
-                                break;
-                            }
-                        }
-                    }
-
-                    if (foundClose && (eraseStyle == EraseStyle.InlineMarkup)) {
-                        if ((i != length() - 1) &&
-                                (INLINE_MARKUP_DELIMITERS.indexOf(charAt(i + close.length())) == -1)) {
-                            foundClose = false;
-                        }
-                    }
-
-                    if (foundClose) {
-                        switch (eraseStyle) {
-                            case All:
-                                erase(enclosureStart, (i - enclosureStart) + close.length());
-                                break;
-                            case Markers:
-                            case InlineMarkup:
-                                erase(enclosureStart, open.length());
-                                erase(i, close.length());
-                                break;
-                            case PreserveLabel:
-                                if (lastCommaPosition != -1) {
-                                    erase(enclosureStart, (lastCommaPosition + 1) - enclosureStart);
-                                    erase(i, close.length());
-                                }
-                                else {
-                                    erase(enclosureStart, open.length());
-                                    erase(i, close.length());
-                                }
-                                break;
-                            case CloseMarkerContainsDelimiters:
-                                erase(enclosureStart, (i == length() - 1)
-                                        ? (length() - enclosureStart)
-                                        : (i - enclosureStart));
-                                break;
-                            case None:
-                                break;
-                        }
-                        inEnclosure = false;
-                        lastCommaPosition = -1;
-                    }
-                    else if (charAt(i) == ',') {
-                        lastCommaPosition = i;
-                    }
-                }
-            }
-        }
-        return firstEnclosurePosition;
-    }
-
     /**
      * Return the length of the line
      *
      * @return length of the line
      */
     public int length() {
-        return text.size();
+        return characters.size();
     }
 
     /**
@@ -348,42 +136,30 @@ public class Line {
      * @return extracted character
      */
     public char charAt(int i, boolean includeInvalid) {
-        if ((i >= 0) && (i < text.size())) {
+        if ((i >= 0) && (i < characters.size())) {
             if (escaped.get(i)) {
                 return ESCAPED_CHARACTER_VALUE;
             }
             if (includeInvalid || valid.get(i)) {
-                return text.get(i);
+                return characters.get(i);
             }
         }
         return 0;
     }
 
     /**
-     * Return the offset for the character at the given position
+     * Does the line start with the given string?
      *
-     * @param i index
-     * @return offset
+     * @param s string to test
+     * @return true if the string starts with the specified string
      */
-    public int getOffset(int i) {
-        if (i >= 0) {
-            if (i < offsets.size()) {
-                return offsets.get(i);
-            }
-            else {
-                return offsets.size();
+    public boolean startsWith(String s) {
+        for (int i = 0; i < s.length(); i++) {
+            if (charAt(i) != s.charAt(i)) {
+                return false;
             }
         }
-        return 0;
-    }
-
-    /**
-     * Return the original line number for this line
-     *
-     * @return line number
-     */
-    public int getLineNo() {
-        return lineNo;
+        return true;
     }
 
     /**
@@ -393,8 +169,8 @@ public class Line {
      * @return raw character at the specified position
      */
     public char rawCharAt(int i) {
-        if ((i >= 0) && (i < text.size())) {
-            return text.get(i);
+        if ((i >= 0) && (i < characters.size())) {
+            return characters.get(i);
         }
         return ' ';
     }
@@ -406,7 +182,7 @@ public class Line {
      * @return true if the character at the given position valid
      */
     public boolean isValid(int i) {
-        if ((i >= 0) && (i < text.size())) {
+        if ((i >= 0) && (i < characters.size())) {
             return valid.get(i);
         }
         return false;
@@ -418,8 +194,8 @@ public class Line {
      * @return true if it's empty
      */
     public boolean isEmpty() {
-        for (int i = 0; i < text.size(); i++) {
-            if (!Character.isWhitespace(text.get(i)))
+        for (int i = 0; i < characters.size(); i++) {
+            if (!Character.isWhitespace(characters.get(i)))
                 if (valid.get(i)) {
                     return false;
                 }
@@ -429,19 +205,220 @@ public class Line {
     }
 
     /**
-     * Does the line start with the given string?
+     * Return the offset for the character at the given position
      *
-     * @param s string to test
-     * @return true if the string starts with the specified string
+     * @param i index
+     * @return offset
      */
-    public boolean startsWith(String s) {
-        for (int i = 0; i < s.length(); i++) {
-            if (charAt(i) != s.charAt(i)) {
-                return false;
+    public int getOffset(int i) {
+        if (i >= 0) {
+            if (i < offsets.size()) {
+                return offsets.get(i);
+            }
+            else {
+                return offsets.size();
             }
         }
+        return 0;
+    }
 
-        return true;
+    /**
+     * Is this line a repeating set of the same character?
+     *
+     * @return true is all same character
+     */
+    public boolean isAllSameCharacter() {
+        return allSameCharacter;
+    }
+
+    /**
+     * Has this line been completely erased?
+     *
+     * @return true if erased
+     */
+    public boolean isErased() {
+        return erased;
+    }
+
+    /**
+     * Is this line in a block?
+     *
+     * @return true is it's in a block
+     */
+    public boolean isInBlock() {
+        return inBlock;
+    }
+
+
+    /**
+     * Is this line the start of a new list item?
+     *
+     * @return true if it's list start
+     */
+    public boolean isListStart() {
+        return listStart;
+    }
+
+    /**
+     * Return the list level for this line, or zero if it is not in a list.
+     *
+     * @return the list level, or zero if not in a list
+     */
+    public int getListLevel() {
+        return listLevel;
+    }
+
+    /**
+     * If the line is a section header, return the section level,
+     * or zero if the line is not a section header
+     *
+     * @return the section level or zero if not a section header
+     */
+    public int getSectionLevel() {
+        return sectionLevel;
+    }
+
+    public void setInBlock(boolean inBlock) {
+        this.inBlock = inBlock;
+    }
+
+    public void setSectionLevel(int newSectionLevel) {
+        this.sectionLevel = newSectionLevel;
+    }
+
+    public void setListLevel(int newListLevel) {
+        this.listLevel = newListLevel;
+    }
+
+    public void setListStart(boolean listStart) {
+        this.listStart = listStart;
+    }
+
+    public String getText() {
+        return text;
+    }
+
+    /**
+     * Erase the open and close markers, and optionally all the text inside them
+     * Returns the position of the first enclosure or -1 if no enclosure was found
+     *
+     * @param open open marker to be erased
+     * @param close close marker to be erased
+     * @param eraseStyle erase style
+     * @return position of first enclosure
+     */
+    public int eraseEnclosure(String open,
+                              String close,
+                              EraseStyle eraseStyle) {
+        boolean inEnclosure = false;
+        int firstEnclosurePosition = -1;
+        int lastCommaPosition = -1;
+        int enclosureStart = 0;
+        for (int i = 0; i < length(); i++) {
+            if (!valid.get(i)) {
+                continue;
+            }
+            if (!inEnclosure) {
+                // look for the open string
+                boolean foundOpen = true;
+                for (int j = 0; j < open.length(); j++) {
+                    if (charAt(i + j) != open.charAt(j)) {
+                        foundOpen = false;
+                        break;
+                    }
+                }
+                // inline requires start of line or a space before the marker
+                if (foundOpen && (eraseStyle == EraseStyle.InlineMarkup)) {
+                    if ((i != 0) &&
+                            inlineMarkupDelimiters.length() > 0 &&
+                            (inlineMarkupDelimiters.indexOf(charAt(i - 1)) == -1)) {
+                        foundOpen = false;
+                    }
+                }
+                if (foundOpen) {
+                    enclosureStart = i;
+                    inEnclosure = true;
+                    firstEnclosurePosition = i;
+                }
+            }
+            else {
+                // look for the close string
+                boolean foundClose = true;
+                if (eraseStyle == EraseStyle.CloseMarkerContainsDelimiters) {
+                    foundClose = (i == length() - 1) || (close.indexOf(charAt(i)) != -1);
+                }
+                else {
+                    for (int j = 0; j < close.length(); j++) {
+                        if (charAt(i + j) != close.charAt(j)) {
+                            foundClose = false;
+                            break;
+                        }
+                    }
+                }
+
+                if (foundClose && (eraseStyle == EraseStyle.InlineMarkup)) {
+                    if ((i != length() - 1) && inlineMarkupDelimiters.length() > 0 &&
+                            (inlineMarkupDelimiters.indexOf(charAt(i + close.length())) == -1)) {
+                        foundClose = false;
+                    }
+                }
+
+                if (foundClose) {
+                    eraseWithStyle(open, close, eraseStyle, lastCommaPosition, enclosureStart, i);
+                    inEnclosure = false;
+                    lastCommaPosition = -1;
+                }
+                else if (charAt(i) == ',') {
+                    lastCommaPosition = i;
+                }
+            }
+        }
+        return firstEnclosurePosition;
+    }
+
+    private void eraseWithStyle(String open,
+                                String close,
+                                EraseStyle eraseStyle,
+                                int lastCommaPosition,
+                                int enclosureStart,
+                                int start) {
+        switch (eraseStyle) {
+            case All:
+                erase(enclosureStart, (start - enclosureStart) + close.length());
+                break;
+            case Markers:
+            case InlineMarkup:
+                erase(enclosureStart, open.length());
+                erase(start, close.length());
+                break;
+            case PreserveLabel:
+                if (lastCommaPosition != -1) {
+                    erase(enclosureStart, (lastCommaPosition + 1) - enclosureStart);
+                    erase(start, close.length());
+                }
+                else {
+                    erase(enclosureStart, open.length());
+                    erase(start, close.length());
+                }
+                break;
+            case CloseMarkerContainsDelimiters:
+                erase(enclosureStart, (start == length() - 1)
+                        ? (length() - enclosureStart)
+                        : (start - enclosureStart));
+                break;
+            case None:
+                break;
+        }
+    }
+
+
+    /**
+     * Return the original line number for this line
+     *
+     * @return line number
+     */
+    public int getLineNo() {
+        return lineNo;
     }
 
     /**
@@ -458,12 +435,12 @@ public class Line {
     @Override
     public String toString() {
         String result = "";
-        for (int i = 0; i < text.size(); i++) {
+        for (int i = 0; i < characters.size(); i++) {
             if (valid.get(i)) {
-                result += text.get(i);
+                result += characters.get(i);
             }
             else {
-                result += "·" + text.get(i);
+                result += "·" + characters.get(i);
             }
         }
         return (erased ? "X" : " ") +
@@ -475,4 +452,5 @@ public class Line {
                 " " +
                 result;
     }
+
 }
diff --git a/redpen-core/src/main/java/cc/redpen/parser/common/LineParser.java b/redpen-core/src/main/java/cc/redpen/parser/common/LineParser.java
new file mode 100644
index 00000000..fd780d02
--- /dev/null
+++ b/redpen-core/src/main/java/cc/redpen/parser/common/LineParser.java
@@ -0,0 +1,137 @@
+/*
+ * redpen: a text inspection tool
+ * Copyright (c) 2014-2015 Recruit Technologies Co., Ltd. and contributors
+ * (see CONTRIBUTORS.md)
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package cc.redpen.parser.common;
+
+import cc.redpen.RedPenException;
+import cc.redpen.model.Document;
+import cc.redpen.parser.BaseDocumentParser;
+import cc.redpen.parser.SentenceExtractor;
+import cc.redpen.tokenizer.RedPenTokenizer;
+
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+
+public abstract class LineParser extends BaseDocumentParser {
+
+    /**
+     * Target line of parser
+     */
+    public class TargetLine {
+        // target line
+        public Line line;
+        // previous line of target line
+        public Line previousLine;
+        // next line of target line
+        public Line nextLine;
+        // fist character of target line
+        public char firstChar;
+        // second character of target line
+        public char secondChar;
+
+        public TargetLine(Line line, Line previousLine,
+                          Line nextLine) {
+            this.line = line;
+            this.previousLine = previousLine;
+            this.nextLine = nextLine;
+            this.firstChar = line.charAt(0);
+            this.secondChar = line.charAt(1);
+        }
+    }
+
+    @Override
+    public Document parse(InputStream io, Optional<String> fileName,
+                          SentenceExtractor sentenceExtractor, RedPenTokenizer tokenizer) throws RedPenException {
+        Document.DocumentBuilder documentBuilder = Document.builder(tokenizer);
+        fileName.ifPresent(documentBuilder::setFileName);
+
+        Model model = new Model(sentenceExtractor);
+
+        // add the lines from the input to the abstract model
+        this.populateModel(model, io);
+
+        // convert the model to a redpen document
+        this.convertModel(model, documentBuilder);
+
+        return documentBuilder.build();
+    }
+
+    /**
+     * Conver the parser's model to the RedPen document model
+     *
+     * @param model model to convert
+     * @param builder doc builder
+     */
+    protected void convertModel(Model model, Document.DocumentBuilder builder) {
+        model.rewind();
+
+        // add a header if there isn't one in the model
+        if ((model.getCurrentLine() != null) && (model.getCurrentLine().getSectionLevel() == 0)) {
+            builder.addSection(0);
+        }
+
+        while (model.isMore()) {
+            // skip blank lines
+            while (model.isMore() && model.getCurrentLine().isEmpty()) {
+                model.getNextLine();
+            }
+            if (!model.isMore()) { return; } // no contents
+
+            // check for new sections
+            if (model.getCurrentLine().getSectionLevel() > 0) {
+                builder.addSection(
+                        model.getCurrentLine().getSectionLevel(),
+                        model.convertToSentences(model.getCurrentLine())
+                );
+                model.getNextLine();
+            }
+            // check for a list item
+            else if (model.getCurrentLine().isListStart()) {
+                List<Line> listElementLines = new ArrayList<>();
+                int listLevel = model.getCurrentLine().getListLevel();
+
+                // add the list start line
+                listElementLines.add(model.getCurrentLine());
+
+                // test the following lines to see if they continue this list item
+                model.getNextLine();
+                while (model.isMore() &&
+                        !model.getCurrentLine().isListStart() &&
+                        (model.getCurrentLine().getListLevel() == listLevel)) {
+                    listElementLines.add(model.getCurrentLine());
+                    model.getNextLine();
+                }
+                builder.addListElement(listLevel, model.convertToSentences(listElementLines));
+            }
+            // process a paragraph
+            else {
+                List<Line> paragraphLines = new ArrayList<>();
+                // current line can't be empty, so this loop will enter at least once
+                while (model.isMore() && !model.getCurrentLine().isEmpty()) {
+                    paragraphLines.add(model.getCurrentLine());
+                    model.getNextLine();
+                }
+                builder.addParagraph();
+                model.convertToSentences(paragraphLines).forEach(builder::addSentence);
+            }
+        }
+    }
+
+    protected abstract void populateModel(Model model, InputStream io);
+}
diff --git a/redpen-core/src/main/java/cc/redpen/parser/asciidoc/Model.java b/redpen-core/src/main/java/cc/redpen/parser/common/Model.java
similarity index 99%
rename from redpen-core/src/main/java/cc/redpen/parser/asciidoc/Model.java
rename to redpen-core/src/main/java/cc/redpen/parser/common/Model.java
index 6c056ddf..dc59c50a 100644
--- a/redpen-core/src/main/java/cc/redpen/parser/asciidoc/Model.java
+++ b/redpen-core/src/main/java/cc/redpen/parser/common/Model.java
@@ -15,7 +15,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-package cc.redpen.parser.asciidoc;
+package cc.redpen.parser.common;
 
 import cc.redpen.model.Sentence;
 import cc.redpen.parser.LineOffset;
@@ -30,9 +30,7 @@ import java.util.List;
  */
 public class Model {
     private static final Line EMPTY_LINE = new Line("", 0);
-
     private List<Line> lines = new ArrayList<>();
-
     private int lineIndex = 0;
     private SentenceExtractor sentenceExtractor;
 
diff --git a/redpen-core/src/main/java/cc/redpen/parser/review/ReVIEWLine.java b/redpen-core/src/main/java/cc/redpen/parser/review/ReVIEWLine.java
new file mode 100644
index 00000000..98217372
--- /dev/null
+++ b/redpen-core/src/main/java/cc/redpen/parser/review/ReVIEWLine.java
@@ -0,0 +1,55 @@
+/*
+ * redpen: a text inspection tool
+ * Copyright (c) 2014-2015 Recruit Technologies Co., Ltd. and contributors
+ * (see CONTRIBUTORS.md)
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package cc.redpen.parser.review;
+
+import cc.redpen.parser.common.Line;
+
+public class ReVIEWLine extends Line {
+    /**
+     * Construct a line using the supplied string
+     *
+     * @param str    the text of the line
+     * @param lineno the original line number
+     */
+    public ReVIEWLine(String str, int lineno) {
+        super(str, lineno);
+        this.inlineMarkupDelimiters = "";
+        if (!str.isEmpty()) {
+            for (int i = 0; i < str.length(); i++) {
+                char ch = str.charAt(i);
+                if ((i < str.length() - 1) && (ch == '\\')) {
+                    i++;
+                    ch = str.charAt(i);
+                    escaped.add(true);
+                }
+                else {
+                    escaped.add(false);
+                }
+                offsets.add(i);
+                characters.add(ch);
+                valid.add(true);
+            }
+        }
+
+        // trim the end
+        while (!characters.isEmpty() &&
+                Character.isWhitespace(characters.get(characters.size() - 1))) {
+            characters.remove(characters.size() - 1);
+        }
+    }
+}
diff --git a/redpen-core/src/main/java/cc/redpen/parser/review/ReVIEWParser.java b/redpen-core/src/main/java/cc/redpen/parser/review/ReVIEWParser.java
new file mode 100644
index 00000000..97f3ac7e
--- /dev/null
+++ b/redpen-core/src/main/java/cc/redpen/parser/review/ReVIEWParser.java
@@ -0,0 +1,297 @@
+/**
+ * redpen: a text inspection tool
+ * Copyright (c) 2014-2015 Recruit Technologies Co., Ltd. and contributors
+ * (see CONTRIBUTORS.md)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package cc.redpen.parser.review;
+
+import cc.redpen.parser.common.Line;
+import cc.redpen.parser.common.LineParser;
+import cc.redpen.parser.common.Model;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+
+public class ReVIEWParser extends LineParser {
+    private static final Logger LOG = LoggerFactory.getLogger(ReVIEWParser.class);
+
+    /**
+     * current parser state
+     */
+    private class State {
+        // are we in a block
+        public boolean inBlock = false;
+        // are we in a list?
+        public boolean inList = false;
+        // should we erase lines within the current block?
+        public boolean eraseBlock = true;
+        // the sort of block we are in
+        public String type;
+    }
+
+    class ReVIEWBlock {
+        public String type = "";
+        public List<String> properties = new ArrayList<>();
+        public boolean isOpen = false;
+    }
+
+    static Pattern DIGIT_PATTERN = Pattern.compile("^\\s+[0-9]+\\.");
+
+    protected void populateModel(Model model, InputStream io) {
+        State state = new State();
+        BufferedReader reader = createReader(io);
+
+        int lineno = 0;
+        try {
+            // add the lines to the model
+            while (true) {
+                String line = reader.readLine();
+                if (line == null) {
+                    break;
+                }
+                lineno++;
+                model.add(new ReVIEWLine(line, lineno));
+            }
+            reader.close();
+
+            for (model.rewind(); model.isMore(); model.getNextLine()) {
+                processLine(model.getCurrentLine(), model, state);
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+            LOG.error("Exception when parsing Re:VIEW file", e);
+        }
+        if (LOG.isDebugEnabled()) {
+            LOG.debug("Re:VIEW parser model (X=erased line,[=block,section-listlevel-lineno,*=list item):\n" + model.toString());
+        }
+    }
+
+    private void processLine(Line line, Model model, State state) {
+        if (line.isErased()) { return; }
+
+        TargetLine target = new TargetLine(line,
+                model.getLine(line.getLineNo() - 1),
+                model.getLine(line.getLineNo() + 1));
+
+        if (state.inList && (line.getListLevel() == 0)) {
+            line.setListLevel(target.previousLine.getListLevel());
+        }
+
+        // check for block end
+        if (state.inBlock) {
+            if (target.line.startsWith("//}") && target.line.length() == 3) {
+                state.inBlock = false;
+            }
+            line.erase();
+        }
+
+        // test for various block starts
+        if ((!state.inBlock) && target.line.startsWith("//")) {
+            ReVIEWBlock block = parseBlock(line);
+            line.erase();
+            state.inBlock = block.isOpen;
+        }
+
+        // handling comments
+        if (target.line.startsWith("#@#")) {
+            line.erase();
+        }
+
+        // enclosed inline markups
+        line.eraseEnclosure("@<list>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<code>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<img>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<table>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<fn>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<chap>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<title>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<chapref>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<bou>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<kw>{", "}", Line.EraseStyle.Markers); // TODO: extract keyword only
+        line.eraseEnclosure("@<chapter>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<ruby>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<ami>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<b>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<i>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<strong>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<em>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<tt>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<tti>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<ttb>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<u>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<br>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<m>{", "}", Line.EraseStyle.All);
+        line.eraseEnclosure("@<icon>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<uchar>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<href>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<column>{", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<raw>{html|", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<raw>{latex|", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<raw>{idgxml|", "}", Line.EraseStyle.Markers);
+        line.eraseEnclosure("@<raw>{top|", "}", Line.EraseStyle.Markers);
+
+        //opening annotation for preprocessor
+        line.eraseEnclosure("#@warn(", ")", Line.EraseStyle.All);
+        line.eraseEnclosure("@comment(", ")", Line.EraseStyle.All);
+        line.eraseEnclosure("#@mapfile(", ")", Line.EraseStyle.All);
+        line.eraseEnclosure("#@maprange(", ")", Line.EraseStyle.All);
+        line.eraseEnclosure("#@mapoutput(", ")", Line.EraseStyle.All);
+
+        //closing annotation for preprocessor
+        if (target.line.startsWith("#@end")) {
+            line.erase();
+        }
+
+        // handling section
+        int headerIndent = 0;
+        while (line.charAt(headerIndent) == '=') {
+            headerIndent++;
+        }
+        if (line.charAt(headerIndent) == '[') { // column?
+            while (line.charAt(headerIndent) != ']'
+                    && line.charAt(headerIndent) != 0) {
+                headerIndent++;
+            }
+            headerIndent++;
+        }
+        if ((headerIndent > 0) && (line.charAt(headerIndent) == ' ')) {
+            line.erase(0, headerIndent + 1);
+            line.setSectionLevel(headerIndent);
+        }
+        if (target.line.startsWith("===[/column]")) { // closing..
+            line.erase();
+        }
+
+        // list
+        if (!state.inBlock && isListElement(line, target.nextLine)) {
+            state.inList = true;
+        }
+
+        // a blank line will cancel any list element we are in
+        if (state.inList && (line.length() == 0)) {
+            state.inList = false;
+            line.setListLevel(0);
+        }
+    }
+
+    /**
+     * Does the give line start a list?
+     *
+     * @param line
+     * @return
+     */
+    private boolean isListElement(Line line, Line nextLine) {
+        if (isNormalList(line)) return true;
+
+        // is numbered list?
+        if (isDigitList(line)) return true;
+
+        // is labeled list?
+        if (isLabeledList(line, nextLine)) return true;
+
+        return false;
+    }
+
+    private boolean isLabeledList(Line line, Line nextLine) {
+        int spacePos = 0;
+        while(' ' == line.charAt(spacePos)) { spacePos++; }
+
+        ;
+
+        if (line.charAt(spacePos) == ':') {
+            nextLine.setListLevel(1);
+            nextLine.setListStart(true);
+            line.erase();
+            return true;
+        }
+        return false;
+    }
+
+    private boolean isDigitList(Line line) {
+        Matcher m = DIGIT_PATTERN.matcher(line.getText());
+        if (m.find()) {
+            int dotPos = line.getText().indexOf(".");
+            line.setListLevel(1);
+            line.setListStart(true);
+            line.erase(0, dotPos);
+            while (Character.isWhitespace(line.charAt(dotPos))) {
+                line.erase(dotPos, 1);
+                dotPos++;
+            }
+            return true;
+        }
+        return false;
+    }
+
+    private boolean isNormalList(Line line) {
+        int pos = 0;
+        while(' ' == line.charAt(pos)) { pos++; }
+        if (pos == 0) { return false; }
+
+        // is the first non-space character a suitable list marker?
+        if ("*".indexOf(line.charAt(pos)) != -1) {
+            char listMarker = line.charAt(pos);
+            int level = 1; pos++;
+            while (line.charAt(pos) == listMarker) {
+                pos++;
+                level++;
+            }
+
+            if (Character.isWhitespace(line.charAt(pos))) {
+                line.setListLevel(level);
+                line.setListStart(true);
+                line.erase(0, pos);
+                while (Character.isWhitespace(line.charAt(pos))) {
+                    line.erase(pos, 1);
+                    pos++;
+                }
+                return true;
+            }
+        }
+        return false;
+    }
+
+    ReVIEWBlock parseBlock(Line line) {
+        ReVIEWBlock block = new ReVIEWBlock();
+        String text = line.getText();
+        // detect type
+        int openIdx = text.indexOf("[");
+        if (openIdx > 0) {
+            block.type = text.substring(2, openIdx);
+            // detect properties
+            int closeIdx = text.indexOf("]");
+            while(closeIdx > 0) {
+                block.properties.add(text.substring(openIdx+1, closeIdx));
+                openIdx = text.indexOf("[", openIdx+1);
+                closeIdx = text.indexOf("]", closeIdx+1);
+            }
+        } else {
+            block.type = text.substring(2, text.indexOf("{"));
+        }
+
+        // detect open block
+        if (text.indexOf("{") > 0) {
+            block.isOpen = true;
+        }
+        return block;
+    }
+}


Test patch:
diff --git a/redpen-core/src/test/java/cc/redpen/parser/asciidoc/AsciiDocTest.java b/redpen-core/src/test/java/cc/redpen/parser/asciidoc/AsciiDocTest.java
index 4186a1d8..fa71ef69 100644
--- a/redpen-core/src/test/java/cc/redpen/parser/asciidoc/AsciiDocTest.java
+++ b/redpen-core/src/test/java/cc/redpen/parser/asciidoc/AsciiDocTest.java
@@ -19,6 +19,7 @@ package cc.redpen.parser.asciidoc;
 
 import cc.redpen.config.Configuration;
 import cc.redpen.parser.SentenceExtractor;
+import cc.redpen.parser.common.Model;
 import org.junit.Test;
 
 import java.io.ByteArrayInputStream;
@@ -125,12 +126,8 @@ public class AsciiDocTest {
         assertEquals(modelText, model.toString());
     }
 
-
-    /**
-     * Test the line class
-     */
     @Test
-    public void testLine() {
+    public void testEraseEnclosure() {
         String testLine = "A line *with* an [enclosure] and one with a [enclosure,with a description] " +
                 "and then an URL like http://fred.fish/";
 
@@ -142,36 +139,36 @@ public class AsciiDocTest {
                 "  0-0-001: A line *with* an ·[enclosure·] and one with a ·[enclosure,with a description·] and then an URL like http://fred.fish/",
                 "  0-0-001: A line *with* an ·[enclosure·] and one with a ·[·e·n·c·l·o·s·u·r·e·,with a description·] and then an URL like http://fred.fish/"
         };
-        Line line;
+        AsciiDocLine line;
 
         // this erases nothing since the close tag is not found
-        line = new Line(testLine, 1);
-        line.eraseEnclosure("[", "]]", Line.EraseStyle.All);
+        line = new AsciiDocLine(testLine, 1);
+        line.eraseEnclosure("[", "]]", AsciiDocLine.EraseStyle.All);
         assertEquals(expectedResults[0], line.toString());
 
         // すべてが消去されますか？
-        line = new Line(testLine, 1);
-        line.eraseEnclosure("[", "]", Line.EraseStyle.All);
+        line = new AsciiDocLine(testLine, 1);
+        line.eraseEnclosure("[", "]", AsciiDocLine.EraseStyle.All);
         assertEquals(expectedResults[1], line.toString());
 
         // Test a string of delimiters as an ending tag
-        line = new Line(testLine, 1);
-        line.eraseEnclosure("http://", " ,[", Line.EraseStyle.CloseMarkerContainsDelimiters);
+        line = new AsciiDocLine(testLine, 1);
+        line.eraseEnclosure("http://", " ,[", AsciiDocLine.EraseStyle.CloseMarkerContainsDelimiters);
         assertEquals(expectedResults[2], line.toString());
 
         // Test inline markup erasure
-        line = new Line(testLine, 1);
-        line.eraseEnclosure("*", "*", Line.EraseStyle.InlineMarkup);
+        line = new AsciiDocLine(testLine, 1);
+        line.eraseEnclosure("*", "*", AsciiDocLine.EraseStyle.InlineMarkup);
         assertEquals(expectedResults[3], line.toString());
 
         // Test just removing the markers
-        line = new Line(testLine, 1);
-        line.eraseEnclosure("[", "]", Line.EraseStyle.Markers);
+        line = new AsciiDocLine(testLine, 1);
+        line.eraseEnclosure("[", "]", AsciiDocLine.EraseStyle.Markers);
         assertEquals(expectedResults[4], line.toString());
 
         // Test just removing the delimter tags
-        line = new Line(testLine, 1);
-        line.eraseEnclosure("[", "]", Line.EraseStyle.PreserveLabel);
+        line = new AsciiDocLine(testLine, 1);
+        line.eraseEnclosure("[", "]", AsciiDocLine.EraseStyle.PreserveLabel);
         assertEquals(expectedResults[5], line.toString());
     }
 }
diff --git a/redpen-core/src/test/java/cc/redpen/parser/common/LineTest.java b/redpen-core/src/test/java/cc/redpen/parser/common/LineTest.java
new file mode 100644
index 00000000..6c2db296
--- /dev/null
+++ b/redpen-core/src/test/java/cc/redpen/parser/common/LineTest.java
@@ -0,0 +1,22 @@
+/*
+ * redpen: a text inspection tool
+ * Copyright (c) 2014-2015 Recruit Technologies Co., Ltd. and contributors
+ * (see CONTRIBUTORS.md)
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cc.redpen.parser.common;
+
+public class LineTest {
+}
diff --git a/redpen-core/src/test/java/cc/redpen/parser/review/ReVIEWLineTest.java b/redpen-core/src/test/java/cc/redpen/parser/review/ReVIEWLineTest.java
new file mode 100644
index 00000000..a22e7045
--- /dev/null
+++ b/redpen-core/src/test/java/cc/redpen/parser/review/ReVIEWLineTest.java
@@ -0,0 +1,45 @@
+/*
+ * redpen: a text inspection tool
+ * Copyright (c) 2014-2015 Recruit Technologies Co., Ltd. and contributors
+ * (see CONTRIBUTORS.md)
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package cc.redpen.parser.review;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class ReVIEWLineTest {
+    @Test
+    public void testEraseInlineEnclosure() {
+        String testLine = "A line with an @<b>{enclosure}.";
+        String expectedResult =
+                "  0-0-001: A line with an ·@·<·b·>·{enclosure·}.";
+        ReVIEWLine line = new ReVIEWLine(testLine, 1);
+        line.eraseEnclosure("@<b>{", "}", ReVIEWLine.EraseStyle.InlineMarkup);
+        assertEquals(expectedResult, line.toString());
+    }
+
+    @Test
+    public void testEraseWholeEnclosure() {
+        String testLine = "This is a comment: @<comment>{TODO}.";
+        String expectedResult =
+                "  0-0-001: This is a comment: ·@·<·c·o·m·m·e·n·t·>·{·T·O·D·O·}.";
+        ReVIEWLine line = new ReVIEWLine(testLine, 1);
+        line.eraseEnclosure("@<comment>{", "}", ReVIEWLine.EraseStyle.All);
+        assertEquals(expectedResult, line.toString());
+    }
+}
diff --git a/redpen-core/src/test/java/cc/redpen/parser/review/ReVIEWParserTest.java b/redpen-core/src/test/java/cc/redpen/parser/review/ReVIEWParserTest.java
new file mode 100644
index 00000000..29deb7fa
--- /dev/null
+++ b/redpen-core/src/test/java/cc/redpen/parser/review/ReVIEWParserTest.java
@@ -0,0 +1,270 @@
+/*
+ * redpen: a text inspection tool
+ * Copyright (c) 2014-2015 Recruit Technologies Co., Ltd. and contributors
+ * (see CONTRIBUTORS.md)
+ * <p>
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package cc.redpen.parser.review;
+
+import cc.redpen.RedPenException;
+import cc.redpen.config.Configuration;
+import cc.redpen.model.Document;
+import cc.redpen.model.Paragraph;
+import cc.redpen.model.Section;
+import cc.redpen.parser.DocumentParser;
+import cc.redpen.parser.LineOffset;
+import cc.redpen.parser.SentenceExtractor;
+import org.junit.Test;
+
+import java.io.UnsupportedEncodingException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.Assert.*;
+
+public class ReVIEWParserTest {
+    @Test
+    public void ParseBlock() throws Exception {
+        String sample = "//list[yml_sample][my.yml]{";
+        ReVIEWParser parser = new ReVIEWParser();
+        ReVIEWParser.ReVIEWBlock block = parser.parseBlock(new ReVIEWLine(sample, 0));
+        assertEquals("list", block.type);
+        assertEquals(2, block.properties.size());
+        assertEquals("yml_sample", block.properties.get(0));
+        assertEquals("my.yml", block.properties.get(1));
+        assertTrue(block.isOpen);
+    }
+
+    @Test
+    public void ParseBlockWithoutProperties() throws Exception {
+        String sample = "//lead{";
+        ReVIEWParser parser = new ReVIEWParser();
+        ReVIEWParser.ReVIEWBlock block = parser.parseBlock(new ReVIEWLine(sample, 0));
+        assertEquals("lead", block.type);
+        assertEquals(0, block.properties.size());
+        assertTrue(block.isOpen);
+    }
+
+
+    @Test
+    public void testMultipleShortLine() {
+        String sampleText = "Tokyu\n" +
+                "is a good\n" +
+                "railway company. But there\n" +
+                "are competitors.";
+        Document doc = createFileContent(sampleText);
+        Section firstSections = doc.getSection(0);
+        Paragraph firstParagraph = firstSections.getParagraph(0);
+        assertEquals(2, firstParagraph.getNumberOfSentences());
+
+        assertEquals("Tokyu is a good railway company.", doc.getSection(0).getParagraph(0).getSentence(0).getContent());
+        assertEquals(1, doc.getSection(0).getParagraph(0).getSentence(0).getLineNumber());
+        assertEquals(0, doc.getSection(0).getParagraph(0).getSentence(0).getStartPositionOffset());
+        assertEquals(32, doc.getSection(0).getParagraph(0).getSentence(0).getOffsetMapSize());
+
+        assertEquals(" But there are competitors.", doc.getSection(0).getParagraph(0).getSentence(1).getContent());
+        assertEquals(3, doc.getSection(0).getParagraph(0).getSentence(1).getLineNumber());
+        assertEquals(16, doc.getSection(0).getParagraph(0).getSentence(1).getStartPositionOffset());
+    }
+
+    @Test
+    public void testRemoveTextDecoration() throws UnsupportedEncodingException {
+        String sampleText = "About @<b>{Gekioko}.\n";
+        Document doc = createFileContent(sampleText);
+        assertEquals(1, doc.size());
+        assertEquals("About Gekioko.", doc.getSection(0).getParagraph(0).getSentence(0).getContent());
+    }
+
+    @Test
+    public void testGenerateDocumentWithList() {
+        String sampleText = "There are several railway companies in Japan as follows.\n";
+        sampleText += "\n";
+        sampleText += " * Tokyu\n";
+        sampleText += " ** Toyoko Line\n";
+        sampleText += " ** Denentoshi Line\n";
+        sampleText += " * Keio\n";
+        sampleText += "  * Odakyu\n";
+
+        Document doc = createFileContent(sampleText);
+        assertEquals(5, doc.getSection(0).getListBlock(0).getNumberOfListElements());
+        assertEquals("Tokyu", doc.getSection(0).getListBlock(0).getListElement(0).getSentence(0).getContent());
+        assertEquals(1, doc.getSection(0).getListBlock(0).getListElement(0).getLevel());
+        assertEquals(3, doc.getSection(0).getListBlock(0).getListElement(0).getSentence(0).getLineNumber());
+        assertEquals(3, doc.getSection(0).getListBlock(0).getListElement(0).getSentence(0).getStartPositionOffset());
+
+        assertEquals("Toyoko Line", doc.getSection(0).getListBlock(0).getListElement(1).getSentence(0).getContent());
+        assertEquals(2, doc.getSection(0).getListBlock(0).getListElement(1).getLevel());
+        assertEquals(4, doc.getSection(0).getListBlock(0).getListElement(1).getSentence(0).getLineNumber());
+        assertEquals(4, doc.getSection(0).getListBlock(0).getListElement(1).getSentence(0).getStartPositionOffset());
+
+        assertEquals("Denentoshi Line", doc.getSection(0).getListBlock(0).getListElement(2).getSentence(0).getContent());
+        assertEquals(2, doc.getSection(0).getListBlock(0).getListElement(2).getLevel());
+        assertEquals(5, doc.getSection(0).getListBlock(0).getListElement(2).getSentence(0).getLineNumber());
+        assertEquals(4, doc.getSection(0).getListBlock(0).getListElement(2).getSentence(0).getStartPositionOffset());
+
+        assertEquals("Keio", doc.getSection(0).getListBlock(0).getListElement(3).getSentence(0).getContent());
+        assertEquals(1, doc.getSection(0).getListBlock(0).getListElement(3).getLevel());
+        assertEquals(6, doc.getSection(0).getListBlock(0).getListElement(3).getSentence(0).getLineNumber());
+        assertEquals(3, doc.getSection(0).getListBlock(0).getListElement(3).getSentence(0).getStartPositionOffset());
+
+        assertEquals("Odakyu", doc.getSection(0).getListBlock(0).getListElement(4).getSentence(0).getContent());
+        assertEquals(1, doc.getSection(0).getListBlock(0).getListElement(4).getLevel());
+        assertEquals(7, doc.getSection(0).getListBlock(0).getListElement(4).getSentence(0).getLineNumber());
+        assertEquals(4, doc.getSection(0).getListBlock(0).getListElement(4).getSentence(0).getStartPositionOffset());
+    }
+
+    @Test
+    public void testLabelledList() {
+        String sampleText = "= SampleDoc\n" +
+                "\n" +
+                "== 用語定義\n" +
+                "ユビキタス言語を定義します。\n" +
+                "\n" +
+                "  : Some word\n" +
+                "なにかの意味をのせて用例をのせます。\n" +
+                "\n" +
+                " : リリース\n" +
+                "ソフトウェアを顧客に提供することです。\n" +
+                "\n" +
+                " : redpen\n" +
+                "RedPen はオープンソースの校正ツールです。RedPen は技術文書が文書規約に従って書かれているかを自動検査します。 現在の RedPen 日本語ドキュメントは十分検査されておりません。校正にはもう少々時間がかかる予定です。誤りなど見つかりましたら、https://github.com/redpen-cc/redpen-doc-ja に Issue 登録しておしらせ頂けると幸いです。";
+
+        Document doc = createFileContent(sampleText);
+        assertEquals(3, doc.getSection(1).getListBlock(0).getNumberOfListElements());
+        assertEquals("なにかの意味をのせて用例をのせます。", doc.getSection(1).getListBlock(0).getListElement(0).getSentence(0).getContent());
+        assertEquals(13, doc.getSection(1).getListBlock(0).getListElement(2).getSentence(0).getLineNumber());
+
+    }
+
+    @Test
+    public void testComment() {
+        String sampleText = "#@# BLAH BLAH" +
+                "\n" +
+                "Potato";
+
+        Document doc = createFileContent(sampleText);
+
+        for (Section section : doc) {
+            for (Paragraph paragraph : section.getParagraphs()) {
+                paragraph.getSentences().forEach(sentence -> {
+                    assertEquals("Potato", sentence.getContent());
+                });
+            }
+        }
+    }
+
+    @Test
+    public void testTable() {
+        String sampleText = "#@# BLAH BLAH" +
+                "\n" +
+                "//table[envvars][重要な環境変数]{" +
+                "名前            意味" +
+                "-------------------------------------------------------------" +
+                "PATH            コマンドの存在するディレクトリ" +
+                "TERM            使っている端末の種類。linux・kterm・vt100など" +
+                "//}";
+
+        Document doc = createFileContent(sampleText);
+        for (Section section : doc) {
+            for (Paragraph paragraph : section.getParagraphs()) {
+                paragraph.getSentences().forEach(sentence -> {
+                    assertEquals("Potato", sentence.getContent());
+                });
+            }
+        }
+    }
+
+    @Test
+    public void testSectionHeader() throws UnsupportedEncodingException {
+        String sampleText = "= About @<i>{Gekioko}.\n\n" +
+                "Gekioko means angry.";
+
+        Document doc = createFileContent(sampleText);
+        assertEquals(1, doc.size());
+        assertEquals("About Gekioko.", doc.getSection(0).getHeaderContent(0).getContent());
+    }
+
+    @Test
+    public void testColumnSectionHeader() throws UnsupportedEncodingException {
+        String sampleText = "==[column] About Gekioko. \n\n";
+        Document doc = createFileContent(sampleText);
+        assertEquals(1, doc.size());
+        assertEquals("About Gekioko.", doc.getSection(0).getHeaderContent(0).getContent());
+    }
+
+    @Test
+    public void testInvalidColumnSectionHeader() throws UnsupportedEncodingException {
+        String sampleText = "==[column About Gekioko.\n\n"; // NOTE: no "]"
+        Document doc = createFileContent(sampleText);
+        assertEquals(1, doc.size());
+        assertEquals("==[column About Gekioko.", doc.getSection(0).getParagraph(0).getSentence(0).getContent());
+    }
+
+    @Test
+    public void testDocumentWithBoldWord() {
+        String sampleText = "It is a @<b>{good} day.";
+        Document doc = createFileContent(sampleText);
+        Section firstSections = doc.getSection(0);
+        Paragraph firstParagraph = firstSections.getParagraph(0);
+        assertEquals("It is a good day.", firstParagraph.getSentence(0).getContent());
+        List<LineOffset> expectedOffsets = initializeMappingTable(
+                new LineOffset(1, 0),
+                new LineOffset(1, 1),
+                new LineOffset(1, 2),
+                new LineOffset(1, 3),
+                new LineOffset(1, 4),
+                new LineOffset(1, 5),
+                new LineOffset(1, 6),
+                new LineOffset(1, 7),
+                new LineOffset(1, 13),
+                new LineOffset(1, 14),
+                new LineOffset(1, 15),
+                new LineOffset(1, 16),
+                new LineOffset(1, 18),
+                new LineOffset(1, 19),
+                new LineOffset(1, 20),
+                new LineOffset(1, 21),
+                new LineOffset(1, 22));
+        assertEquals(expectedOffsets.size(), firstParagraph.getSentence(0).getOffsetMapSize());
+        for (int i = 0; i < expectedOffsets.size(); i++) {
+            assertEquals(expectedOffsets.get(i), firstParagraph.getSentence(0).getOffset(i).get());
+        }
+    }
+
+    private static List<LineOffset> initializeMappingTable(LineOffset... offsets) {
+        List<LineOffset> offsetTable = new ArrayList<>();
+        for (LineOffset offset : offsets) {
+            offsetTable.add(offset);
+        }
+        return offsetTable;
+    }
+
+
+    private Document createFileContent(String inputDocumentString) {
+        DocumentParser parser = DocumentParser.REVIEW;
+        Document doc = null;
+        try {
+            Configuration configuration = Configuration.builder().build();
+            doc = parser.parse(
+                    inputDocumentString,
+                    new SentenceExtractor(configuration.getSymbolTable()),
+                    configuration.getTokenizer());
+        } catch (RedPenException e) {
+            e.printStackTrace();
+            fail();
+        }
+        return doc;
+    }
+
+}