Repository: redpen-cc/redpen
Pull request #541 (2016-02-02 08:52:30)

Language detection in Java
Language detection is more advanced now and also detects Japanese variants: zenkaku, zenkaku2 and hankaku.


Patch:
diff --git a/redpen-core/src/main/java/cc/redpen/tokenizer/WhiteSpaceTokenizer.java b/redpen-core/src/main/java/cc/redpen/tokenizer/WhiteSpaceTokenizer.java
index c1a1b188..a7d7755a 100644
--- a/redpen-core/src/main/java/cc/redpen/tokenizer/WhiteSpaceTokenizer.java
+++ b/redpen-core/src/main/java/cc/redpen/tokenizer/WhiteSpaceTokenizer.java
@@ -21,15 +21,13 @@ import java.util.ArrayList;
 import java.util.List;
 import java.util.regex.Pattern;
 
-import static java.lang.Character.isWhitespace;
-
 public class WhiteSpaceTokenizer implements RedPenTokenizer {
 
     private static final Pattern[] BLACKLIST_TOKEN_PATTERNS = new Pattern[]{
             Pattern.compile("^[-+#$€£¥]?\\d+(\\.\\d+)?[%€¥¢₽]?$") // a number [+-]n[.n]
     };
 
-    private static final String DELIMITERS = " \u00A0\t\n\r?!,:;.()\u2014\"";
+    private static final String DELIMITERS = " \t\n\r?!,:;.()\u2014\"";
 
     public WhiteSpaceTokenizer() {
     }
@@ -48,7 +46,7 @@ public class WhiteSpaceTokenizer implements RedPenTokenizer {
                 if (isSuitableToken(surface)) {
                     tokens.add(new TokenElement(surface, tags, offset));
                 }
-                if (!isWhitespace(ch) && ch != '\u00A0') {
+                if (!Character.isWhitespace(ch)) {
                     tokens.add(new TokenElement(String.valueOf(ch), tags, i));
                 }
                 surface = "";
diff --git a/redpen-core/src/main/java/cc/redpen/util/LanguageDetector.java b/redpen-core/src/main/java/cc/redpen/util/LanguageDetector.java
new file mode 100644
index 00000000..6588eaec
--- /dev/null
+++ b/redpen-core/src/main/java/cc/redpen/util/LanguageDetector.java
@@ -0,0 +1,28 @@
+package cc.redpen.util;
+
+import static cc.redpen.util.StringUtils.isProbablyJapanese;
+import static java.lang.Math.min;
+
+public class LanguageDetector {
+  public String detectLanguage(String text) {
+    if (!hasJapaneseCharacters(text)) return "en";
+
+    boolean zenkaku = text.indexOf('。') >= 0 || text.indexOf('、') >= 0 || text.indexOf('！') >= 0 || text.indexOf('？') >= 0;
+    boolean zenkaku2 = text.indexOf('．') >= 0 || text.indexOf('，') >= 0;
+    boolean hankaku = text.indexOf('.') >= 0 || text.indexOf(',') >= 0 || text.indexOf('!') >= 0 || text.indexOf('?') >= 0;
+
+    return zenkaku ? "ja" :
+           zenkaku2 ? "ja.zenkaku2" :
+           hankaku ? "ja.hankaku":
+           "ja";
+  }
+
+  private boolean hasJapaneseCharacters(String text) {
+    char[] chars = text.toCharArray();
+    for (int i = 0; i < min(chars.length, 100); i++) {
+      char c = chars[i];
+      if (isProbablyJapanese(c)) return true;
+    }
+    return false;
+  }
+}
diff --git a/redpen-core/src/main/java/cc/redpen/util/StringUtils.java b/redpen-core/src/main/java/cc/redpen/util/StringUtils.java
index 29dfe42b..ffd156ea 100644
--- a/redpen-core/src/main/java/cc/redpen/util/StringUtils.java
+++ b/redpen-core/src/main/java/cc/redpen/util/StringUtils.java
@@ -17,9 +17,24 @@
  */
 package cc.redpen.util;
 
+import static java.lang.Character.UnicodeBlock.*;
+
 public class StringUtils {
     public static boolean isKatakana(char c) {
-        return Character.UnicodeBlock.of(c) == Character.UnicodeBlock.KATAKANA;
+        return Character.UnicodeBlock.of(c) == KATAKANA;
+    }
+
+    public static boolean isHiragana(char c) {
+        return Character.UnicodeBlock.of(c) == HIRAGANA;
+    }
+
+    public static boolean isCJK(char c) {
+        return Character.UnicodeBlock.of(c) == CJK_UNIFIED_IDEOGRAPHS;
+    }
+
+    public static boolean isProbablyJapanese(char c) {
+        Character.UnicodeBlock block = Character.UnicodeBlock.of(c);
+        return block == KATAKANA || block == HIRAGANA || block == CJK_UNIFIED_IDEOGRAPHS;
     }
 
     public static boolean isBasicLatin(char c) {
diff --git a/redpen-core/src/main/java/cc/redpen/validator/sentence/SymbolWithSpaceValidator.java b/redpen-core/src/main/java/cc/redpen/validator/sentence/SymbolWithSpaceValidator.java
index c47aee57..930f56a4 100644
--- a/redpen-core/src/main/java/cc/redpen/validator/sentence/SymbolWithSpaceValidator.java
+++ b/redpen-core/src/main/java/cc/redpen/validator/sentence/SymbolWithSpaceValidator.java
@@ -49,24 +49,20 @@ public class SymbolWithSpaceValidator extends Validator {
         char target = symbol.getValue();
         int position = sentenceStr.indexOf(target);
         if (position != -1) {
-            String key = "";
-
             if (position > 0 && symbol.isNeedBeforeSpace()
                     && !Character.isWhitespace(sentenceStr.charAt(position - 1))) {
-                key = "Before";
-            }
+                addLocalizedErrorWithPosition(sentence,
+                        position,
+                        position + 1,
+                        sentenceStr.charAt(position));
 
-            if (position < sentenceStr.length() - 1
+            } else if (position < sentenceStr.length() - 1
                     && symbol.isNeedAfterSpace()
                     && !Character.isWhitespace(sentenceStr.charAt(position + 1))) {
-                key += "After";
-            }
-
-            if (!key.isEmpty()) {
-                addLocalizedErrorWithPosition(key, sentence,
-                  position,
-                  position + 1,
-                  sentenceStr.charAt(position));
+                addLocalizedErrorWithPosition(sentence,
+                        position,
+                        position + 1,
+                        position);
             }
         }
         return null;
diff --git a/redpen-core/src/main/resources/cc/redpen/validator/sentence/error-messages.properties b/redpen-core/src/main/resources/cc/redpen/validator/sentence/error-messages.properties
index c1a1c9ec..171abb4c 100644
--- a/redpen-core/src/main/resources/cc/redpen/validator/sentence/error-messages.properties
+++ b/redpen-core/src/main/resources/cc/redpen/validator/sentence/error-messages.properties
@@ -20,9 +20,7 @@ QuotationValidator.RightExist=Right quotation mark is missing.
 QuotationValidator.LeftExist=Left quotation mark is missing.
 SpaceBetweenAlphabeticalWordValidator.Before=Space not present before an alphabetical word.
 SpaceBetweenAlphabeticalWordValidator.After=Space not present after an alphabetical word.
-SymbolWithSpaceValidator.Before=Need whitespace before symbol "{0}".
-SymbolWithSpaceValidator.After=Need whitespace after symbol "{0}".
-SymbolWithSpaceValidator.BeforeAfter=Need whitespace before and after symbol "{0}".
+SymbolWithSpaceValidator=Need whitespace before or after symbol "{0}".
 DoubledWordValidator=Found repeated word "{0}".
 SuccessiveWordValidator=Found word "{0}" repeated twice in succession.
 HankakuKanaValidator=Found Hankaku Kana character "{0}".
diff --git a/redpen-core/src/main/resources/cc/redpen/validator/sentence/error-messages_ja.properties b/redpen-core/src/main/resources/cc/redpen/validator/sentence/error-messages_ja.properties
index cdeace17..803b3b70 100644
--- a/redpen-core/src/main/resources/cc/redpen/validator/sentence/error-messages_ja.properties
+++ b/redpen-core/src/main/resources/cc/redpen/validator/sentence/error-messages_ja.properties
@@ -20,9 +20,7 @@ QuotationValidator.RightExist=\u53F3\u30AF\u30A9\u30FC\u30C8\u304C\u5B58\u5728\u
 QuotationValidator.LeftExist=\u5DE6\u30AF\u30A9\u30FC\u30C8\u304C\u5B58\u5728\u3057\u307E\u305B\u3093\u3002
 SpaceBetweenAlphabeticalWordValidator.Before=\u30A2\u30EB\u30D5\u30A1\u30D9\u30C3\u30C8\u5358\u8A9E\u306E\u524D\u306B\u30B9\u30DA\u30FC\u30B9\u304C\u5B58\u5728\u3057\u307E\u305B\u3093\u3002
 SpaceBetweenAlphabeticalWordValidator.After=\u30A2\u30EB\u30D5\u30A1\u30D9\u30C3\u30C8\u5358\u8A9E\u306E\u5F8C\u306B\u30B9\u30DA\u30FC\u30B9\u304C\u5B58\u5728\u3057\u307E\u305B\u3093\u3002
-SymbolWithSpaceValidator.Before=\u30B7\u30F3\u30DC\u30EB\u306E\u524D\u306B\u30B9\u30DA\u30FC\u30B9\u304C\u5FC5\u8981\u3067\u3059 ({0})\u3002
-SymbolWithSpaceValidator.After=\u30B7\u30F3\u30DC\u30EB\u306E\u524D\u306B\u30B9\u30DA\u30FC\u30B9\u304C\u5FC5\u8981\u3067\u3059 ({0})\u3002
-SymbolWithSpaceValidator.BeforeAfter=\u30B7\u30F3\u30DC\u30EB\u306E\u524D\u306B\u30B9\u30DA\u30FC\u30B9\u304C\u5FC5\u8981\u3067\u3059 ({0})\u3002
+SymbolWithSpaceValidator=\u30B7\u30F3\u30DC\u30EB\u306E\u524D\u306B\u30B9\u30DA\u30FC\u30B9\u304C\u5FC5\u8981\u3067\u3059 ({0})\u3002
 DoubledWordValidator=\u4E00\u6587\u306B\u4E8C\u56DE\u4EE5\u4E0A\u5229\u7528\u3055\u308C\u3066\u3044\u308B\u5358\u8A9E "{0}" \u304C\u307F\u3064\u304B\u308A\u307E\u3057\u305F\u3002
 SuccessiveWordValidator=\u5358\u8A9E "{0}" \u306F\u9023\u7D9A\u3057\u3066\u4F7F\u7528\u3055\u308C\u3066\u3044\u307E\u3059\u3002
 HankakuKanaValidator=\u534A\u89D2\u30AB\u30CA\u306E\u6587\u5B57 "{0}" \u304C\u898B\u3064\u304B\u308A\u307E\u3057\u305F\u3002
diff --git a/redpen-server/src/main/java/cc/redpen/server/api/RedPenResource.java b/redpen-server/src/main/java/cc/redpen/server/api/RedPenResource.java
index 7cccfab1..afdb815f 100644
--- a/redpen-server/src/main/java/cc/redpen/server/api/RedPenResource.java
+++ b/redpen-server/src/main/java/cc/redpen/server/api/RedPenResource.java
@@ -27,8 +27,10 @@ import cc.redpen.formatter.Formatter;
 import cc.redpen.model.Document;
 import cc.redpen.parser.DocumentParser;
 import cc.redpen.util.FormatterUtils;
+import cc.redpen.util.LanguageDetector;
 import cc.redpen.validator.ValidationError;
 import org.apache.wink.common.annotations.Workspace;
+import org.json.JSONException;
 import org.json.JSONObject;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -37,7 +39,6 @@ import javax.servlet.ServletContext;
 import javax.ws.rs.*;
 import javax.ws.rs.core.Context;
 import javax.ws.rs.core.MediaType;
-import javax.ws.rs.core.Response;
 import java.util.HashMap;
 import java.util.Iterator;
 import java.util.List;
@@ -60,6 +61,20 @@ public class RedPenResource {
     @Context
     private ServletContext context;
 
+    /**
+     * Detect language of document
+     *
+     * @param document       the source text of the document
+     */
+    @Path("/language")
+    @POST
+    @Produces(MediaType.APPLICATION_JSON)
+    @WinkAPIDescriber.Description("Detect language of document")
+    public JSONObject detectLanguage(@FormParam("document") @DefaultValue("") String document) throws JSONException {
+        String language = new LanguageDetector().detectLanguage(document);
+        return new JSONObject().put("key", language);
+    }
+
     /**
      * Validate a source document posted from a form
      *
@@ -75,7 +90,7 @@ public class RedPenResource {
     @POST
     @Produces(MediaType.APPLICATION_JSON)
     @WinkAPIDescriber.Description("Validate a document and return any redpen errors")
-    public Response validateDocument(@FormParam("document") @DefaultValue("") String document,
+    public String validateDocument(@FormParam("document") @DefaultValue("") String document,
                                      @FormParam("documentParser") @DefaultValue(DEFAULT_DOCUMENT_PARSER) String documentParser,
                                      @FormParam("lang") @DefaultValue(DEFAULT_CONFIGURATION) String lang,
                                      @FormParam("format") @DefaultValue(DEFAULT_FORMAT) String format,
@@ -97,8 +112,7 @@ public class RedPenResource {
             throw new RedPenException("Unsupported format: " + format + " - please use xml, plain, plain2, json or json2");
         }
 
-        String responseJSON = formatter.format(parsedDocument, errors);
-        return Response.ok().entity(responseJSON).build();
+        return formatter.format(parsedDocument, errors);
     }
 
 
@@ -120,7 +134,7 @@ public class RedPenResource {
     @Consumes(MediaType.APPLICATION_JSON)
     @Produces(MediaType.APPLICATION_JSON)
     @WinkAPIDescriber.Description("Process a redpen JSON validation request and returns any redpen errors")
-    public Response validateDocumentJSON(JSONObject requestJSON) throws RedPenException {
+    public String validateDocumentJSON(JSONObject requestJSON) throws RedPenException {
 
         LOG.info("Validating document using JSON request");
         String documentParser = getOrDefault(requestJSON, "documentParser", DEFAULT_DOCUMENT_PARSER);
@@ -199,8 +213,7 @@ public class RedPenResource {
             throw new RedPenException("Unsupported format: " + format + " - please use xml, plain, plain2, json or json2");
         }
 
-        String responseJSON = formatter.format(parsedDocument, errors);
-        return Response.ok().entity(responseJSON).build();
+        return formatter.format(parsedDocument, errors);
     }
 
     private String getOrDefault(JSONObject json, String property, String defaultValue) {
diff --git a/redpen-server/src/main/webapp/js/redpen.js b/redpen-server/src/main/webapp/js/redpen.js
index 888c49ea..004bba1b 100644
--- a/redpen-server/src/main/webapp/js/redpen.js
+++ b/redpen-server/src/main/webapp/js/redpen.js
@@ -41,22 +41,21 @@ var redpen = (function ($) {
                 if (callback) {
                     callback(data);
                 }
-            },
-            dataType: "json"
+            }
         }).fail(function (err) {
-                console.log(err);
-            });
+            console.error(err);
+        });
     };
 
     this.setBaseUrl = function(url) {
         baseUrl = url;
     };
 
-    // placeholder (and cheap client-side implementation) of a detect-language function
     this.detectLanguage = function (text, callback) {
         if (text) {
-            var japanese = (text.indexOf('。') != -1) || (text.indexOf('、') != -1) || (text.indexOf('は') != -1);
-            callback(japanese ? 'ja' : 'en');
+            doAPICall('document/language', {document: text}, function(data) {
+                callback(data.key);
+            }, 'POST');
         }
     };
 


Test patch:
diff --git a/redpen-core/src/test/java/cc/redpen/tokenizer/WhiteSpaceTokenizerTest.java b/redpen-core/src/test/java/cc/redpen/tokenizer/WhiteSpaceTokenizerTest.java
index 6f9a7b46..5e68a5b3 100644
--- a/redpen-core/src/test/java/cc/redpen/tokenizer/WhiteSpaceTokenizerTest.java
+++ b/redpen-core/src/test/java/cc/redpen/tokenizer/WhiteSpaceTokenizerTest.java
@@ -27,7 +27,7 @@ public class WhiteSpaceTokenizerTest {
     @Test
     public void testTokenize() {
         RedPenTokenizer tokenizer = new WhiteSpaceTokenizer();
-        List<TokenElement> results = tokenizer.tokenize("this is a\u00A0pen");
+        List<TokenElement> results = tokenizer.tokenize("this is a pen");
         assertEquals(4, results.size());
         assertEquals("this", results.get(0).getSurface());
         assertEquals(0, results.get(0).getTags().size());
diff --git a/redpen-core/src/test/java/cc/redpen/util/LanguageDetectorTest.java b/redpen-core/src/test/java/cc/redpen/util/LanguageDetectorTest.java
new file mode 100644
index 00000000..06e737ec
--- /dev/null
+++ b/redpen-core/src/test/java/cc/redpen/util/LanguageDetectorTest.java
@@ -0,0 +1,50 @@
+package cc.redpen.util;
+
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+
+public class LanguageDetectorTest {
+  LanguageDetector detector = new LanguageDetector();
+
+  @Test
+  public void englishIsDefault() throws Exception {
+    assertEquals("en", detector.detectLanguage("Hello there!"));
+  }
+
+  @Test
+  public void japaneseIsDetectedForKatakana() throws Exception {
+    assertEquals("ja", detector.detectLanguage("コンピューター"));
+  }
+
+  @Test
+  public void japaneseIsDetectedForHiragana() throws Exception {
+    assertEquals("ja", detector.detectLanguage("はなぢ"));
+  }
+
+  @Test
+  public void japaneseIsDetectedForKanji() throws Exception {
+    assertEquals("ja", detector.detectLanguage("日本"));
+    assertEquals("ja", detector.detectLanguage("最近"));
+  }
+
+  @Test
+  public void zenkakuIsDefaultJapanese() throws Exception {
+    assertEquals("ja", detector.detectLanguage("最近利用されているソフトウェアの中には"));
+  }
+
+  @Test
+  public void zenkaku2IsDetectedUsingPunctuation() throws Exception {
+    assertEquals("ja.zenkaku2", detector.detectLanguage("こんにちは世界．"));
+  }
+
+  @Test
+  public void zenkakuIsSelectedEvenIfSomeZenkaku2SymbolsArePresent() throws Exception {
+    assertEquals("ja", detector.detectLanguage("こんにちは世界． こんにちは世界。"));
+  }
+
+  @Test
+  public void hankakuIsSelectedIfAsciiPunctuationIsUsedWithJapaneseSymbols() throws Exception {
+    assertEquals("ja.hankaku", detector.detectLanguage("こんにちは世界!"));
+  }
+}
\ No newline at end of file
diff --git a/redpen-core/src/test/java/cc/redpen/validator/sentence/SymbolWithSpaceValidatorTest.java b/redpen-core/src/test/java/cc/redpen/validator/sentence/SymbolWithSpaceValidatorTest.java
index 2e4b9d35..d0a37d46 100644
--- a/redpen-core/src/test/java/cc/redpen/validator/sentence/SymbolWithSpaceValidatorTest.java
+++ b/redpen-core/src/test/java/cc/redpen/validator/sentence/SymbolWithSpaceValidatorTest.java
@@ -23,11 +23,12 @@ import cc.redpen.config.Configuration;
 import cc.redpen.config.Symbol;
 import cc.redpen.config.ValidatorConfiguration;
 import cc.redpen.model.Document;
+import cc.redpen.model.Sentence;
 import cc.redpen.parser.DocumentParser;
 import cc.redpen.parser.LineOffset;
 import cc.redpen.parser.SentenceExtractor;
-import cc.redpen.validator.BaseValidatorTest;
 import cc.redpen.validator.ValidationError;
+import junit.framework.Assert;
 import org.junit.Test;
 
 import java.util.ArrayList;
@@ -35,96 +36,113 @@ import java.util.List;
 import java.util.Map;
 
 import static cc.redpen.config.SymbolType.*;
-import static java.util.Collections.singletonList;
 import static org.junit.Assert.assertEquals;
 
-public class SymbolWithSpaceValidatorTest extends BaseValidatorTest {
-
-    public SymbolWithSpaceValidatorTest() {
-        super("SymbolWithSpace");
-    }
-
+public class SymbolWithSpaceValidatorTest {
     @Test
     public void testNotNeedSpace() throws RedPenException {
-        Document document = prepareSimpleDocument("I like apple/orange");
-
-        config = new Configuration.ConfigurationBuilder()
+        List<Document> documents = new ArrayList<>();
+        documents.add(
+                new Document.DocumentBuilder()
+                        .addSection(1)
+                        .addParagraph()
+                        .addSentence(new Sentence("I like apple/orange", 1))
+                        .build());
+
+        Configuration conf = new Configuration.ConfigurationBuilder()
                 .addValidatorConfig(new ValidatorConfiguration("SymbolWithSpace"))
                 .setLanguage("en")
                 .addSymbol(new Symbol(SLASH, '/'))
                 .build();
 
-        RedPen redPen = new RedPen(config);
-        Map<Document, List<ValidationError>> errors = redPen.validate(singletonList(document));
-        assertEquals(0, errors.get(document).size());
+        RedPen redPen = new RedPen(conf);
+        Map<Document, List<ValidationError>> errors = redPen.validate(documents);
+        Assert.assertEquals(0, errors.get(documents.get(0)).size());
     }
 
     @Test
     public void testNeedAfterSpace() throws RedPenException {
-        Document document = prepareSimpleDocument("I like her:yes it is");
-
-        config = new Configuration.ConfigurationBuilder()
+        List<Document> documents = new ArrayList<>();
+        documents.add(
+                new Document.DocumentBuilder()
+                        .addSection(1)
+                        .addParagraph()
+                        .addSentence(new Sentence("I like her:yes it is", 1))
+                        .build());
+
+        Configuration conf = new Configuration.ConfigurationBuilder()
                 .addValidatorConfig(new ValidatorConfiguration("SymbolWithSpace"))
                 .setLanguage("en")
                 .addSymbol(new Symbol(COLON, ':', "", false, true))
                 .build();
 
-        RedPen redPen = new RedPen(config);
-        Map<Document, List<ValidationError>> errors = redPen.validate(singletonList(document));
-        assertEquals(1, errors.get(document).size());
-        assertEquals("Need whitespace after symbol \":\".", errors.get(document).get(0).getMessage());
-        assertEquals(10, errors.get(document).get(0).getStartPosition().get().offset);
+        RedPen redPen = new RedPen(conf);
+        Map<Document, List<ValidationError>> errors = redPen.validate(documents);
+        Assert.assertEquals(1, errors.get(documents.get(0)).size());
     }
 
     @Test
     public void testNeedBeforeSpace() throws RedPenException {
-        Document document = prepareSimpleDocument("I like her(Nancy) very much.");
-
-        config = new Configuration.ConfigurationBuilder()
+        List<Document> documents = new ArrayList<>();
+        documents.add(
+                new Document.DocumentBuilder()
+                        .addSection(1)
+                        .addParagraph()
+                        .addSentence(new Sentence("I like her(Nancy) very much.", 1))
+                        .build());
+
+        Configuration conf = new Configuration.ConfigurationBuilder()
                 .addValidatorConfig(new ValidatorConfiguration("SymbolWithSpace"))
                 .setLanguage("en")
                 .addSymbol(new Symbol(LEFT_PARENTHESIS, '(', "", true, false))
                 .build();
 
-        RedPen redPen = new RedPen(config);
-        Map<Document, List<ValidationError>> errors = redPen.validate(singletonList(document));
-        assertEquals(1, errors.get(document).size());
-        assertEquals("Need whitespace before symbol \"(\".", errors.get(document).get(0).getMessage());
-        assertEquals(10, errors.get(document).get(0).getStartPosition().get().offset);
+        RedPen redPen = new RedPen(conf);
+        Map<Document, List<ValidationError>> errors = redPen.validate(documents);
+        Assert.assertEquals(1, errors.get(documents.get(0)).size());
     }
 
     @Test
     public void testNeedSpaceInMultiplePosition() throws RedPenException {
-        Document document = prepareSimpleDocument("I like her(Nancy)very much.");
-
-        config = new Configuration.ConfigurationBuilder()
+        List<Document> documents = new ArrayList<>();
+        documents.add(
+                new Document.DocumentBuilder()
+                        .addSection(1)
+                        .addParagraph()
+                        .addSentence(new Sentence("I like her(Nancy)very much.", 1))
+                        .build());
+
+        Configuration conf = new Configuration.ConfigurationBuilder()
                 .addValidatorConfig(new ValidatorConfiguration("SymbolWithSpace"))
                 .setLanguage("en")
                 .addSymbol(new Symbol(LEFT_PARENTHESIS, '(', "", true, false))
                 .addSymbol(new Symbol(RIGHT_PARENTHESIS, ')', "", false, true))
                 .build();
 
-        RedPen redPen = new RedPen(config);
-        Map<Document, List<ValidationError>> errors = redPen.validate(singletonList(document));
-        assertEquals(2, errors.get(document).size());
-        assertEquals("Need whitespace after symbol \")\".", errors.get(document).get(0).getMessage());
-        assertEquals("Need whitespace before symbol \"(\".", errors.get(document).get(1).getMessage());
+        RedPen redPen = new RedPen(conf);
+        Map<Document, List<ValidationError>> errors = redPen.validate(documents);
+        Assert.assertEquals(2, errors.get(documents.get(0)).size());
     }
 
     @Test
     public void testReturnOnlyOneForHitBothBeforeAndAfter() throws RedPenException {
-        Document document = prepareSimpleDocument("I like 1*10.");
-
-        Configuration config = new Configuration.ConfigurationBuilder()
+        List<Document> documents = new ArrayList<>();
+        documents.add(
+                new Document.DocumentBuilder()
+                        .addSection(1)
+                        .addParagraph()
+                        .addSentence(new Sentence("I like 1*10.", 1))
+                        .build());
+
+        Configuration conf = new Configuration.ConfigurationBuilder()
                 .addValidatorConfig(new ValidatorConfiguration("SymbolWithSpace"))
                 .setLanguage("en")
                 .addSymbol(new Symbol(ASTERISK, '*', "", true, true))
                 .build();
 
-        RedPen redPen = new RedPen(config);
-        Map<Document, List<ValidationError>> errors = redPen.validate(singletonList(document));
-        assertEquals(1, errors.get(document).size());
-        assertEquals("Need whitespace before and after symbol \"*\".", errors.get(document).get(0).getMessage());
+        RedPen redPen = new RedPen(conf);
+        Map<Document, List<ValidationError>> errors = redPen.validate(documents);
+        Assert.assertEquals(1, errors.get(documents.get(0)).size());
     }
 
     @Test
diff --git a/redpen-server/src/test/java/cc/redpen/server/api/RedPenResourceTest.java b/redpen-server/src/test/java/cc/redpen/server/api/RedPenResourceTest.java
index 625652f9..9d5958ca 100644
--- a/redpen-server/src/test/java/cc/redpen/server/api/RedPenResourceTest.java
+++ b/redpen-server/src/test/java/cc/redpen/server/api/RedPenResourceTest.java
@@ -110,6 +110,11 @@ public class RedPenResourceTest extends MockServletInvocationTest {
         }
     }
 
+    public void testDetectLanguage() throws Exception {
+        assertEquals("en", new RedPenResource().detectLanguage("Hello World").getString("key"));
+        assertEquals("ja", new RedPenResource().detectLanguage("こんにちは世界").getString("key"));
+    }
+
     // test helper
     private MockHttpServletRequest constructMockRequest(String method,
                                                         String requestURI,