Repository: redpen-cc/redpen
Pull request #553 (2016-02-29 11:43:33)

PropertiesParser implemented for localization files


Patch:
diff --git a/redpen-core/src/main/java/cc/redpen/model/Sentence.java b/redpen-core/src/main/java/cc/redpen/model/Sentence.java
index 0df9b647..35c57904 100644
--- a/redpen-core/src/main/java/cc/redpen/model/Sentence.java
+++ b/redpen-core/src/main/java/cc/redpen/model/Sentence.java
@@ -76,13 +76,13 @@ public class Sentence implements Serializable {
      * Constructor.
      *
      * @param sentenceContent  content of sentence
-     * @param sentencePosition sentence position
+     * @param lineNum          line number of sentence
      * @param startOffset      offset of the start position in the line
      */
-    public Sentence(String sentenceContent, int sentencePosition, int startOffset) {
+    public Sentence(String sentenceContent, int lineNum, int startOffset) {
         super();
         this.content = sentenceContent;
-        this.lineNumber = sentencePosition;
+        this.lineNumber = lineNum;
         this.isFirstSentence = false;
         this.links = new ArrayList<>();
         this.tokens = new ArrayList<>();
@@ -291,4 +291,8 @@ public class Sentence implements Serializable {
         result = 31 * result + (tokens != null ? tokens.hashCode() : 0);
         return result;
     }
+
+    public List<LineOffset> getOffsetMap() {
+        return offsetMap;
+    }
 }
diff --git a/redpen-core/src/main/java/cc/redpen/parser/BaseDocumentParser.java b/redpen-core/src/main/java/cc/redpen/parser/BaseDocumentParser.java
index 6ef9bcf6..16630148 100644
--- a/redpen-core/src/main/java/cc/redpen/parser/BaseDocumentParser.java
+++ b/redpen-core/src/main/java/cc/redpen/parser/BaseDocumentParser.java
@@ -20,8 +20,6 @@ package cc.redpen.parser;
 import cc.redpen.RedPenException;
 import cc.redpen.model.Document;
 import cc.redpen.tokenizer.RedPenTokenizer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.*;
 import java.nio.charset.StandardCharsets;
@@ -32,9 +30,6 @@ import java.util.Optional;
  * implements the concrete Parser classes.
  */
 public abstract class BaseDocumentParser implements DocumentParser {
-    private static final Logger LOG = LoggerFactory.getLogger(
-            BaseDocumentParser.class);
-
 
     @Override
     public Document parse(String content, SentenceExtractor sentenceExtractor, RedPenTokenizer tokenizer) throws RedPenException {
diff --git a/redpen-core/src/main/java/cc/redpen/parser/DocumentParser.java b/redpen-core/src/main/java/cc/redpen/parser/DocumentParser.java
index 3fbc90c6..78d0108a 100644
--- a/redpen-core/src/main/java/cc/redpen/parser/DocumentParser.java
+++ b/redpen-core/src/main/java/cc/redpen/parser/DocumentParser.java
@@ -84,13 +84,14 @@ public interface DocumentParser {
     Document parse(File file, SentenceExtractor sentenceExtractor, RedPenTokenizer tokenizer)
             throws RedPenException;
 
-    public static final DocumentParser PLAIN = new PlainTextParser();
-    public static final DocumentParser WIKI = new WikiParser();
-    public static final DocumentParser MARKDOWN = new MarkdownParser();
-    public static final DocumentParser LATEX = new LaTeXParser();
-    public static final DocumentParser ASCIIDOC = new AsciiDocParser();
+    DocumentParser PLAIN = new PlainTextParser();
+    DocumentParser WIKI = new WikiParser();
+    DocumentParser MARKDOWN = new MarkdownParser();
+    DocumentParser LATEX = new LaTeXParser();
+    DocumentParser ASCIIDOC = new AsciiDocParser();
+    DocumentParser PROPERTIES = new PropertiesParser();
 
-    public static final Map<String, DocumentParser> PARSER_MAP = Collections.unmodifiableMap(
+    Map<String, DocumentParser> PARSER_MAP = Collections.unmodifiableMap(
             new HashMap<String, DocumentParser>() {
                 {
                     put("PLAIN", PLAIN);
@@ -98,6 +99,7 @@ public interface DocumentParser {
                     put("MARKDOWN", MARKDOWN);
                     put("LATEX", LATEX);
                     put("ASCIIDOC", ASCIIDOC);
+                    put("PROPERTIES", PROPERTIES);
                 }
             });
 
diff --git a/redpen-core/src/main/java/cc/redpen/parser/LaTeXParser.java b/redpen-core/src/main/java/cc/redpen/parser/LaTeXParser.java
index 829e1e78..e0083fae 100644
--- a/redpen-core/src/main/java/cc/redpen/parser/LaTeXParser.java
+++ b/redpen-core/src/main/java/cc/redpen/parser/LaTeXParser.java
@@ -36,8 +36,6 @@ import java.util.Optional;
  */
 class LaTeXParser extends BaseDocumentParser {
 
-    private LaTeXProcessor latexProcessor = new LaTeXProcessor();
-
     LaTeXParser() {
         super();
     }
diff --git a/redpen-core/src/main/java/cc/redpen/parser/LineOffset.java b/redpen-core/src/main/java/cc/redpen/parser/LineOffset.java
index 0d261ac4..198adad8 100644
--- a/redpen-core/src/main/java/cc/redpen/parser/LineOffset.java
+++ b/redpen-core/src/main/java/cc/redpen/parser/LineOffset.java
@@ -34,10 +34,7 @@ public class LineOffset implements Comparable {
 
     @Override
     public String toString() {
-        return "LineOffset{" +
-                "lineNum=" + lineNum +
-                ", offset=" + offset +
-                '}';
+        return "LineOffset(" + lineNum + ", " + offset + ")";
     }
 
     @Override
diff --git a/redpen-core/src/main/java/cc/redpen/parser/PlainTextParser.java b/redpen-core/src/main/java/cc/redpen/parser/PlainTextParser.java
index 37adf59b..aa8aaa2f 100644
--- a/redpen-core/src/main/java/cc/redpen/parser/PlainTextParser.java
+++ b/redpen-core/src/main/java/cc/redpen/parser/PlainTextParser.java
@@ -21,8 +21,6 @@ import cc.redpen.RedPenException;
 import cc.redpen.model.Document;
 import cc.redpen.model.Sentence;
 import cc.redpen.tokenizer.RedPenTokenizer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 import java.io.BufferedReader;
 import java.io.IOException;
@@ -36,7 +34,6 @@ import java.util.Optional;
  * Parser for plain text file.
  */
 final public class PlainTextParser extends BaseDocumentParser implements Serializable {
-    private static final Logger LOG = LoggerFactory.getLogger(PlainTextParser.class);
     private static final long serialVersionUID = -4343255148183552844L;
 
     /**
diff --git a/redpen-core/src/main/java/cc/redpen/parser/PropertiesParser.java b/redpen-core/src/main/java/cc/redpen/parser/PropertiesParser.java
new file mode 100644
index 00000000..ceb92324
--- /dev/null
+++ b/redpen-core/src/main/java/cc/redpen/parser/PropertiesParser.java
@@ -0,0 +1,122 @@
+package cc.redpen.parser;
+
+import cc.redpen.RedPenException;
+import cc.redpen.model.Document;
+import cc.redpen.model.Sentence;
+import cc.redpen.tokenizer.RedPenTokenizer;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Optional;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static java.lang.Character.isWhitespace;
+import static java.lang.Integer.parseInt;
+import static java.util.Collections.emptyList;
+
+/**
+ * Parser for Java properties file format.
+ */
+class PropertiesParser extends BaseDocumentParser {
+
+    @Override
+    public Document parse(InputStream inputStream, Optional<String> fileName, SentenceExtractor sentenceExtractor, RedPenTokenizer tokenizer) throws RedPenException {
+        Document.DocumentBuilder builder = Document.builder(tokenizer);
+        fileName.ifPresent(builder::setFileName);
+
+        try (BufferedReader reader = createReader(inputStream)) {
+            String line;
+            AtomicInteger lineNum = new AtomicInteger(0);
+            while ((line = reader.readLine()) != null) {
+                lineNum.incrementAndGet();
+                int keyStart = skipWhitespace(line, 0);
+                if (keyStart == line.length()) continue;
+                int valueStart = valueOffset(line, keyStart);
+                addSentences(builder, sentenceExtractor, section(line, lineNum, valueStart, reader));
+            }
+        }
+        catch (IOException e) {
+            throw new RedPenException(e);
+        }
+
+        return builder.build();
+    }
+
+    private void addSentences(Document.DocumentBuilder builder, SentenceExtractor sentenceExtractor, ValueWithOffsets value) {
+        if (value == null) return;
+        String text = value.getContent();
+        List<LineOffset> offsets = value.getOffsetMap();
+        builder.addSection(0).addParagraph();
+        while (text.length() > 0) {
+            int end = sentenceExtractor.getSentenceEndPosition(text) + 1;
+            if (end == 0) end = text.length();
+            builder.addSentence(new Sentence(text.substring(0, end), offsets.subList(0, end), emptyList()));
+            text = text.substring(end, text.length());
+            offsets = offsets.subList(end, offsets.size());
+        }
+    }
+
+    private ValueWithOffsets section(String line, AtomicInteger lineNum, int valueStart, BufferedReader reader) throws IOException {
+        int length = line.length();
+        StringBuilder value = new StringBuilder(length);
+        List<LineOffset> offsets = new ArrayList<>(length);
+        for (int i = valueStart; i < length; i++) {
+            char c = line.charAt(i);
+            int offset = i;
+            if (c == '\\') {
+                if (++i == length) {
+                    lineNum.incrementAndGet();
+                    line = reader.readLine();
+                    valueStart = skipWhitespace(line, 0);
+                    offsets.add(new LineOffset(lineNum.get(), valueStart));
+                    Sentence nextLine = section(line, lineNum, valueStart, reader);
+                    if (nextLine == null) continue;
+                    value.append('\n').append(nextLine.getContent());
+                    offsets.addAll(nextLine.getOffsetMap());
+                    continue;
+                }
+
+                c = line.charAt(i);
+                if (c == 'n') c = '\n';
+                else if (c == 't') c = '\t';
+                else if (c == 'f') c = '\f';
+                else if (c == 'r') c = '\r';
+                else if (c == 'u') {
+                    String code = line.substring(i + 1, i + 5);
+                    c = (char)parseInt(code, 16);
+                    i += 4;
+                }
+            }
+            value.append(c);
+            offsets.add(new LineOffset(lineNum.get(), offset));
+        }
+        return value.length() == 0 ? null : new ValueWithOffsets(value.toString(), offsets);
+    }
+
+    private int skipWhitespace(String line, int start) {
+        for (int i = start; i < line.length(); i++)
+            if (!isWhitespace(line.charAt(i))) return i;
+        return line.length();
+    }
+
+    private int valueOffset(String line, int start) {
+        int result = -1;
+        for (int i = start; i < line.length(); i++) {
+            char c = line.charAt(i);
+            if (c == '\\') i++;
+            else if (isWhitespace(c)) result = i;
+            else if (c == ':' || c == '=' || c == '#' || c == '!') {result = i; break;}
+            else if (result >= 0) break;
+        }
+        return skipWhitespace(line, result + 1);
+    }
+
+    private static class ValueWithOffsets extends Sentence {
+        public ValueWithOffsets(String content, List<LineOffset> offsetMap) {
+            super(content, offsetMap, emptyList());
+        }
+    }
+}
diff --git a/redpen-core/src/main/java/cc/redpen/parser/SentenceExtractor.java b/redpen-core/src/main/java/cc/redpen/parser/SentenceExtractor.java
index f7efcabb..bce23950 100644
--- a/redpen-core/src/main/java/cc/redpen/parser/SentenceExtractor.java
+++ b/redpen-core/src/main/java/cc/redpen/parser/SentenceExtractor.java
@@ -55,7 +55,7 @@ public class SentenceExtractor {
      *
      * @param fullStopList set of end of sentence characters
      */
-    SentenceExtractor(char[] fullStopList) {
+    SentenceExtractor(char...fullStopList) {
         this(fullStopList, extractRightQuotations(Configuration.builder().build().getSymbolTable()));
     }
 
@@ -77,7 +77,7 @@ public class SentenceExtractor {
         this.rightQuotationList = rightQuotationList;
         this.fullStopPattern = this.constructEndSentencePattern();
         this.endOfSentenceDetector = new EndOfSentenceDetector(
-                this.fullStopPattern, this.WHITE_WORDS);
+                this.fullStopPattern, WHITE_WORDS);
     }
 
     private static char[] extractPeriods(SymbolTable symbolTable) {
@@ -133,8 +133,7 @@ public class SentenceExtractor {
     }
 
     private static <E> List<E> generateUmList(E... args) {
-        List<E> list = new ArrayList<>(Arrays.asList(args));
-        return list;
+        return new ArrayList<>(Arrays.asList(args));
     }
 
     /**


Test patch:
diff --git a/redpen-core/src/test/java/cc/redpen/MarkdownParserTest.java b/redpen-core/src/test/java/cc/redpen/MarkdownParserTest.java
index cb22eb83..d6f42c96 100644
--- a/redpen-core/src/test/java/cc/redpen/MarkdownParserTest.java
+++ b/redpen-core/src/test/java/cc/redpen/MarkdownParserTest.java
@@ -24,6 +24,7 @@ import cc.redpen.model.Document;
 import cc.redpen.model.ListBlock;
 import cc.redpen.model.Paragraph;
 import cc.redpen.model.Section;
+import cc.redpen.parser.BaseParserTest;
 import cc.redpen.parser.DocumentParser;
 import cc.redpen.parser.LineOffset;
 import cc.redpen.parser.SentenceExtractor;
@@ -38,9 +39,11 @@ import java.util.List;
 
 import static cc.redpen.config.SymbolType.COMMA;
 import static cc.redpen.config.SymbolType.FULL_STOP;
+import static java.util.stream.IntStream.of;
+import static java.util.stream.IntStream.range;
 import static org.junit.Assert.*;
 
-public class MarkdownParserTest {
+public class MarkdownParserTest extends BaseParserTest {
 
     @Before
     public void setup() {
@@ -161,15 +164,14 @@ public class MarkdownParserTest {
         assertEquals(0, secondSection.getParagraph(0).getSentence(1).getStartPositionOffset());
 
         // NOTE: both linebreak and first character of the offsets are 0
-        List<LineOffset> expectedOffsets = initializeMappingTable(
-                new LineOffset(3, 0), // LineBreak NOTE: lineBreak is exist but the offset is "0"
-                new LineOffset(3, 0), // F NOTE: even when there is a Linebreak, sentence start from offset "0".
-                new LineOffset(3, 1)  // u
-        );
+        List<LineOffset> expectedOffsets = offsets(3, of(
+                0, // LineBreak NOTE: lineBreak is exist but the offset is "0"
+                0, // F NOTE: even when there is a Linebreak, sentence start from offset "0".
+                1  // u
+        ));
         for (int i = 0; i < expectedOffsets.size(); i++) {
             assertEquals(expectedOffsets.get(i), secondSection.getParagraph(0).getSentence(1).getOffset(i).get());
         }
-
     }
 
     @Test
@@ -377,29 +379,8 @@ public class MarkdownParserTest {
         assertEquals(firstParagraph.getSentence(0).getContent().length(),
                 firstParagraph.getSentence(0).getOffsetMapSize());
 
-        List<LineOffset> expectedOffsets = initializeMappingTable(
-                new LineOffset(1, 0),
-                new LineOffset(1, 1),
-                new LineOffset(1, 2),
-                new LineOffset(1, 3),
-                new LineOffset(1, 4),
-                new LineOffset(1, 5),
-                new LineOffset(1, 6),
-                new LineOffset(1, 7),
-                new LineOffset(1, 8),
-                new LineOffset(1, 9),
-                new LineOffset(1, 11),
-                new LineOffset(1, 12),
-                new LineOffset(1, 13),
-                new LineOffset(1, 14),
-                new LineOffset(1, 15),
-                new LineOffset(1, 16),
-                new LineOffset(1, 37));
-
-        assertEquals(expectedOffsets.size(), firstParagraph.getSentence(0).getOffsetMapSize());
-        for (int i = 0; i < expectedOffsets.size(); i++) {
-            assertEquals(expectedOffsets.get(i), firstParagraph.getSentence(0).getOffset(i).get());
-        }
+        List<LineOffset> expectedOffsets = offsets(1, range(0, 10), range(11, 17), of(37));
+        assertEquals(expectedOffsets, firstParagraph.getSentence(0).getOffsetMap());
     }
 
     @Test
@@ -471,25 +452,7 @@ public class MarkdownParserTest {
         assertEquals("It is a good day.", firstParagraph.getSentence(0).getContent());
         assertEquals(17, firstParagraph.getSentence(0).getContent().length());
         assertEquals(1, firstParagraph.getSentence(0).getLineNumber());
-        List<LineOffset> expectedOffsets = initializeMappingTable(
-                new LineOffset(1, 0),
-                new LineOffset(1, 1),
-                new LineOffset(1, 2),
-                new LineOffset(1, 3),
-                new LineOffset(1, 4),
-                new LineOffset(1, 5),
-                new LineOffset(1, 6),
-                new LineOffset(1, 7),
-                new LineOffset(1, 8),
-                new LineOffset(1, 9),
-                new LineOffset(1, 10),
-                new LineOffset(1, 11),
-                new LineOffset(1, 12),
-                new LineOffset(1, 13),
-                new LineOffset(1, 14),
-                new LineOffset(1, 15),
-                new LineOffset(1, 16));
-
+        List<LineOffset> expectedOffsets = offsets(1, range(0, 17));
         assertEquals(expectedOffsets.size(), firstParagraph.getSentence(0).getOffsetMapSize());
         for (int i = 0; i < expectedOffsets.size(); i++) {
             assertEquals(expectedOffsets.get(i), firstParagraph.getSentence(0).getOffset(i).get());
@@ -503,25 +466,7 @@ public class MarkdownParserTest {
         Section firstSections = doc.getSection(0);
         Paragraph firstParagraph = firstSections.getParagraph(0);
         assertEquals("It is a good day.", firstParagraph.getSentence(0).getContent());
-        List<LineOffset> expectedOffsets = initializeMappingTable(
-                new LineOffset(1, 0),
-                new LineOffset(1, 1),
-                new LineOffset(1, 2),
-                new LineOffset(1, 3),
-                new LineOffset(1, 4),
-                new LineOffset(1, 5),
-                new LineOffset(1, 6),
-                new LineOffset(1, 7),
-                new LineOffset(1, 9),
-                new LineOffset(1, 10),
-                new LineOffset(1, 11),
-                new LineOffset(1, 12),
-                new LineOffset(1, 14),
-                new LineOffset(1, 15),
-                new LineOffset(1, 16),
-                new LineOffset(1, 17),
-                new LineOffset(1, 18));
-
+        List<LineOffset> expectedOffsets = offsets(1, range(0, 8), range(9, 13), range(14, 19));
         assertEquals(expectedOffsets.size(), firstParagraph.getSentence(0).getOffsetMapSize());
         for (int i = 0; i < expectedOffsets.size(); i++) {
             assertEquals(expectedOffsets.get(i), firstParagraph.getSentence(0).getOffset(i).get());
@@ -535,25 +480,7 @@ public class MarkdownParserTest {
         Section firstSections = doc.getSection(0);
         Paragraph firstParagraph = firstSections.getParagraph(0);
         assertEquals("It is a good day.", firstParagraph.getSentence(0).getContent());
-        List<LineOffset> expectedOffsets = initializeMappingTable(
-                new LineOffset(1, 1),
-                new LineOffset(1, 2),
-                new LineOffset(1, 4),
-                new LineOffset(1, 5),
-                new LineOffset(1, 6),
-                new LineOffset(1, 7),
-                new LineOffset(1, 8),
-                new LineOffset(1, 9),
-                new LineOffset(1, 11),
-                new LineOffset(1, 12),
-                new LineOffset(1, 13),
-                new LineOffset(1, 14),
-                new LineOffset(1, 16),
-                new LineOffset(1, 17),
-                new LineOffset(1, 18),
-                new LineOffset(1, 19),
-                new LineOffset(1, 20));
-
+        List<LineOffset> expectedOffsets = offsets(1, range(1, 3), range(4, 10), range(11, 15), range(16, 21));
         assertEquals(expectedOffsets.size(), firstParagraph.getSentence(0).getOffsetMapSize());
         for (int i = 0; i < expectedOffsets.size(); i++) {
             assertEquals(expectedOffsets.get(i), firstParagraph.getSentence(0).getOffset(i).get());
@@ -567,24 +494,7 @@ public class MarkdownParserTest {
         Section firstSections = doc.getSection(0);
         Paragraph firstParagraph = firstSections.getParagraph(0);
         assertEquals("It is a good day.", firstParagraph.getSentence(0).getContent());
-        List<LineOffset> expectedOffsets = initializeMappingTable(
-                new LineOffset(1, 0),
-                new LineOffset(1, 1),
-                new LineOffset(1, 2),
-                new LineOffset(1, 3),
-                new LineOffset(1, 4),
-                new LineOffset(1, 5),
-                new LineOffset(1, 8),
-                new LineOffset(1, 11),
-                new LineOffset(1, 14),
-                new LineOffset(1, 15),
-                new LineOffset(1, 16),
-                new LineOffset(1, 17),
-                new LineOffset(1, 20),
-                new LineOffset(1, 21),
-                new LineOffset(1, 22),
-                new LineOffset(1, 23),
-                new LineOffset(1, 24));
+        List<LineOffset> expectedOffsets = offsets(1, range(0, 6), of(8), of(11), range(14, 18), range(20, 25));
         assertEquals(expectedOffsets.size(), firstParagraph.getSentence(0).getOffsetMapSize());
         for (int i = 0; i < expectedOffsets.size(); i++) {
             assertEquals(expectedOffsets.get(i), firstParagraph.getSentence(0).getOffset(i).get());
@@ -607,19 +517,7 @@ public class MarkdownParserTest {
         assertEquals(1, lastSection.getHeaderContent(0).getLineNumber());
         assertEquals(2, lastSection.getHeaderContent(0).getStartPositionOffset());
 
-        List<LineOffset> expectedOffsets1 = initializeMappingTable(
-                new LineOffset(1, 2),
-                new LineOffset(1, 3),
-                new LineOffset(1, 4),
-                new LineOffset(1, 5),
-                new LineOffset(1, 6),
-                new LineOffset(1, 7),
-                new LineOffset(1, 8),
-                new LineOffset(1, 9),
-                new LineOffset(1, 10),
-                new LineOffset(1, 11),
-                new LineOffset(1, 12),
-                new LineOffset(1, 13));
+        List<LineOffset> expectedOffsets1 = offsets(1, range(2, 14));
 
         assertEquals(expectedOffsets1.size(), lastSection.getHeaderContent(0).getOffsetMapSize());
         for (int i = 0; i < expectedOffsets1.size(); i++) {
@@ -627,22 +525,7 @@ public class MarkdownParserTest {
                     lastSection.getHeaderContent(0).getOffset(i).get());
         }
 
-        List<LineOffset> expectedOffsets2 = initializeMappingTable(
-                new LineOffset(1, 14),
-                new LineOffset(1, 15),
-                new LineOffset(1, 16),
-                new LineOffset(1, 17),
-                new LineOffset(1, 18),
-                new LineOffset(1, 19),
-                new LineOffset(1, 20),
-                new LineOffset(1, 21),
-                new LineOffset(1, 22),
-                new LineOffset(1, 23),
-                new LineOffset(1, 24),
-                new LineOffset(1, 25),
-                new LineOffset(1, 26),
-                new LineOffset(1, 27),
-                new LineOffset(1, 28));
+        List<LineOffset> expectedOffsets2 = offsets(1, range(14, 29));
 
         assertEquals(" About Saitama.", lastSection.getHeaderContent(1).getContent());
         assertEquals(1, lastSection.getHeaderContent(1).getLineNumber());
@@ -669,18 +552,7 @@ public class MarkdownParserTest {
         assertEquals("About Gunma", lastSection.getHeaderContent(0).getContent());
         assertEquals(2, lastSection.getHeaderContent(0).getStartPositionOffset());
 
-        List<LineOffset> expectedOffsets = initializeMappingTable(
-                new LineOffset(1, 2),
-                new LineOffset(1, 3),
-                new LineOffset(1, 4),
-                new LineOffset(1, 5),
-                new LineOffset(1, 6),
-                new LineOffset(1, 7),
-                new LineOffset(1, 8),
-                new LineOffset(1, 9),
-                new LineOffset(1, 10),
-                new LineOffset(1, 11),
-                new LineOffset(1, 12));
+        List<LineOffset> expectedOffsets = offsets(1, range(2, 13));
         assertEquals(expectedOffsets.size(), lastSection.getHeaderContent(0).getOffsetMapSize());
         for (int i = 0; i < expectedOffsets.size(); i++) {
             assertEquals(expectedOffsets.get(i),
@@ -931,12 +803,4 @@ public class MarkdownParserTest {
         }
         return doc;
     }
-
-    private static List<LineOffset> initializeMappingTable(LineOffset... offsets) {
-        List<LineOffset> offsetTable = new ArrayList<>();
-        for (LineOffset offset : offsets) {
-            offsetTable.add(offset);
-        }
-        return offsetTable;
-    }
 }
diff --git a/redpen-core/src/test/java/cc/redpen/parser/BaseParserTest.java b/redpen-core/src/test/java/cc/redpen/parser/BaseParserTest.java
new file mode 100644
index 00000000..ce248190
--- /dev/null
+++ b/redpen-core/src/test/java/cc/redpen/parser/BaseParserTest.java
@@ -0,0 +1,21 @@
+package cc.redpen.parser;
+
+import java.util.List;
+import java.util.stream.IntStream;
+
+import static java.util.stream.Collectors.toList;
+import static java.util.stream.IntStream.of;
+
+public class BaseParserTest {
+  protected List<LineOffset> offsets(int lineNum, IntStream...streams) {
+    return concat(streams).mapToObj(p -> new LineOffset(lineNum, p)).collect(toList());
+  }
+
+  private IntStream concat(IntStream...streams) {
+    IntStream result = of();
+    for (IntStream stream : streams) {
+      result = IntStream.concat(result, stream);
+    }
+    return result;
+  }
+}
diff --git a/redpen-core/src/test/java/cc/redpen/parser/PropertiesParserTest.java b/redpen-core/src/test/java/cc/redpen/parser/PropertiesParserTest.java
new file mode 100644
index 00000000..721d08a8
--- /dev/null
+++ b/redpen-core/src/test/java/cc/redpen/parser/PropertiesParserTest.java
@@ -0,0 +1,163 @@
+package cc.redpen.parser;
+
+import cc.redpen.RedPenException;
+import cc.redpen.model.Document;
+import cc.redpen.model.Sentence;
+import cc.redpen.tokenizer.WhiteSpaceTokenizer;
+import org.junit.Test;
+
+import java.util.List;
+
+import static java.util.stream.IntStream.of;
+import static java.util.stream.IntStream.range;
+import static org.junit.Assert.assertEquals;
+
+public class PropertiesParserTest extends BaseParserTest {
+  PropertiesParser parser = new PropertiesParser();
+
+  @Test
+  public void keyEqualsValue() throws Exception {
+    Document doc = parse("hello=world");
+    Sentence sentence = doc.getLastSection().getParagraph(0).getSentence(0);
+    assertEquals("world", sentence.getContent());
+  }
+
+  @Test
+  public void keyColonValue() throws Exception {
+    Document doc = parse("hello:world");
+    Sentence sentence = doc.getLastSection().getParagraph(0).getSentence(0);
+    assertEquals("world", sentence.getContent());
+  }
+
+  @Test
+  public void keySpaceValue() throws Exception {
+    Document doc = parse("hello world");
+    Sentence sentence = doc.getLastSection().getParagraph(0).getSentence(0);
+    assertEquals("world", sentence.getContent());
+  }
+
+  @Test
+  public void extraSpaces() throws Exception {
+    Document doc = parse("hello = world ");
+    List<Sentence> sentences = doc.getSection(0).getParagraph(0).getSentences();
+    assertEquals(1, sentences.size());
+    Sentence sentence = sentences.get(0);
+    assertEquals("world ", sentence.getContent());
+    assertEquals(1, sentence.getLineNumber());
+    assertEquals(offsets(1, range(8, 14)), sentence.getOffsetMap());
+  }
+
+  @Test
+  public void extraWhitespace() throws Exception {
+    Document doc = parse("\t hello\f=\tworld ");
+    Sentence sentence = doc.getSection(0).getParagraph(0).getSentences().get(0);
+    assertEquals("world ", sentence.getContent());
+    assertEquals(offsets(1, range(10, 16)), sentence.getOffsetMap());
+  }
+
+  @Test
+  public void threeLines() throws Exception {
+    Document doc = parse("hello = world\r\nworld:earth\n  key=val");
+
+    Sentence sentence = doc.getSection(0).getParagraph(0).getSentence(0);
+    assertEquals("world", sentence.getContent());
+    assertEquals(1, sentence.getLineNumber());
+    assertEquals(offsets(1, range(8, 13)), sentence.getOffsetMap());
+
+    sentence = doc.getSection(1).getParagraph(0).getSentence(0);
+    assertEquals("earth", sentence.getContent());
+    assertEquals(2, sentence.getLineNumber());
+    assertEquals(offsets(2, range(6, 11)), sentence.getOffsetMap());
+
+    sentence = doc.getSection(2).getParagraph(0).getSentence(0);
+    assertEquals("val", sentence.getContent());
+    assertEquals(3, sentence.getLineNumber());
+    assertEquals(offsets(3, range(6, 9)), sentence.getOffsetMap());
+  }
+
+  @Test
+  public void manyWhitespaces() throws Exception {
+    Document doc = parse("hi there the first hi is a key");
+    Sentence sentence = doc.getSection(0).getParagraph(0).getSentence(0);
+    assertEquals("there the first hi is a key", sentence.getContent());
+    assertEquals(offsets(1, range(3, 30)), sentence.getOffsetMap());
+  }
+
+  @Test
+  public void whitespaceInKey() throws Exception {
+    Document doc = parse("two\\ words=value");
+    Sentence sentence = doc.getSection(0).getParagraph(0).getSentence(0);
+    assertEquals("value", sentence.getContent());
+    assertEquals(offsets(1, range(11, 16)), sentence.getOffsetMap());
+  }
+
+  @Test
+  public void multipleDelimiters() throws Exception {
+    Document doc = parse("key=:value");
+    Sentence sentence = doc.getSection(0).getParagraph(0).getSentence(0);
+    assertEquals(":value", sentence.getContent());
+    assertEquals(offsets(1, range(4, 10)), sentence.getOffsetMap());
+  }
+
+  @Test
+  public void emptyLines() throws Exception {
+    Document doc = parse("\nkey=value\n\n\n   \n\n");
+    assertEquals(doc.getSection(0), doc.getLastSection());
+    Sentence sentence = doc.getSection(0).getParagraph(0).getSentence(0);
+    assertEquals("value", sentence.getContent());
+  }
+
+  @Test
+  public void comments() throws Exception {
+    Document doc = parse(" #Hello World\n!Another comment\n#");
+    Sentence sentence = doc.getSection(0).getParagraph(0).getSentence(0);
+    assertEquals("Hello World", sentence.getContent());
+    sentence = doc.getSection(1).getParagraph(0).getSentence(0);
+    assertEquals("Another comment", sentence.getContent());
+  }
+
+  @Test
+  public void valuesAreUnescaped() throws Exception {
+    Document doc = parse("hello=Hello\\ W\\u00F6rld\\t");
+    Sentence sentence = doc.getSection(0).getParagraph(0).getSentence(0);
+    assertEquals("Hello Wörld\t", sentence.getContent());
+    assertEquals(offsets(1, range(6, 12), range(13, 15), range(20, 24)), sentence.getOffsetMap());
+  }
+
+  @Test
+  public void multilineValue() throws Exception {
+    Document doc = parse("hello=Hello\\\n World\\\n\\\n   foo");
+    Sentence sentence = doc.getSection(0).getParagraph(0).getSentence(0);
+    assertEquals("Hello\nWorld\n\nfoo", sentence.getContent());
+    List<LineOffset> offsets = offsets(1, range(6, 11));
+    offsets.addAll(offsets(2, of(1), range(1, 6)));
+    offsets.addAll(offsets(3, of(0)));
+    offsets.addAll(offsets(4, of(3), range(3, 6)));
+    assertEquals(offsets, sentence.getOffsetMap());
+  }
+
+  @Test
+  public void multipleSentences() throws Exception {
+    Document doc = parse("hello=One sentence. Second one! Third one");
+
+    Sentence sentence = doc.getSection(0).getParagraph(0).getSentence(0);
+    assertEquals("One sentence.", sentence.getContent());
+    assertEquals(offsets(1, range(6, 19)), sentence.getOffsetMap());
+
+    sentence = doc.getSection(0).getParagraph(0).getSentence(1);
+    assertEquals(" Second one!", sentence.getContent());
+    assertEquals(offsets(1, range(19, 31)), sentence.getOffsetMap());
+
+    sentence = doc.getSection(0).getParagraph(0).getSentence(2);
+    assertEquals(" Third one", sentence.getContent());
+    assertEquals(offsets(1, range(31, 41)), sentence.getOffsetMap());
+  }
+
+  private Document parse(String content) throws RedPenException {
+    return parser.parse(content, new SentenceExtractor('.', '!'), new WhiteSpaceTokenizer());
+  }
+
+  // todo detect UTF-8 vs ISO-8859-1 files
+
+  // todo test global errors
+}
\ No newline at end of file