Repository: neo4j-contrib/neo4j-apoc-procedures (java)

Issue:
`apoc.text.clean` should not clean umlauts
In case of e.g. German umlauts `clean` just removes those characters. I assume general expectation is convert those to a base form.

I guess the same issue arises with any non standard text character.

http://stackoverflow.com/questions/1008802/converting-symbols-accent-letters-to-english-alphabet has some solution for this.

Before changing this I'd love to hear comments from others.

Pull request:
neo4j-apoc-procedures-78 `apoc.text.clean` should not clean umlauts
fixes #78 

Patch:
diff --git a/src/main/java/apoc/text/Strings.java b/src/main/java/apoc/text/Strings.java
index 20e387b65..820d51cac 100644
--- a/src/main/java/apoc/text/Strings.java
+++ b/src/main/java/apoc/text/Strings.java
@@ -10,6 +10,7 @@ import org.neo4j.procedure.Procedure;
 import java.io.UnsupportedEncodingException;
 import java.net.URLDecoder;
 import java.net.URLEncoder;
+import java.text.Normalizer;
 import java.util.List;
 import java.util.regex.Pattern;
 import java.util.stream.Stream;
@@ -83,8 +84,26 @@ public class Strings {
     }
 
     private static Pattern cleanPattern = Pattern.compile("[^A-Za-z0-9]+");
+    private static Pattern specialCharPattern = Pattern.compile("\\p{IsM}+");
+    private static String[][] UMLAUT_REPLACEMENTS = {
+            { new String("Ä"), "Ae" },
+            { new String("Ü"), "Ue" },
+            { new String("Ö"), "Oe" },
+            { new String("ä"), "ae" },
+            { new String("ü"), "ue" },
+            { new String("ö"), "oe" },
+            { new String("ß"), "ss" }
+    };
+
 
     private static String removeNonWordCharacters(String s) {
-        return cleanPattern.matcher(s).replaceAll("").toLowerCase();
+
+        String result = s ;
+        for (int i=0; i<UMLAUT_REPLACEMENTS.length; i++) {
+            result = result.replace(UMLAUT_REPLACEMENTS[i][0], UMLAUT_REPLACEMENTS[i][1]);
+        }
+        result = Normalizer.normalize(result, Normalizer.Form.NFD);
+        String tmp2 = specialCharPattern.matcher(result).replaceAll("");
+        return cleanPattern.matcher(tmp2).replaceAll("").toLowerCase();
     }
 }


Test patch:
diff --git a/src/test/java/apoc/text/StringCleanTest.java b/src/test/java/apoc/text/StringCleanTest.java
new file mode 100644
index 000000000..e15a29196
--- /dev/null
+++ b/src/test/java/apoc/text/StringCleanTest.java
@@ -0,0 +1,64 @@
+package apoc.text;
+
+import apoc.util.TestUtil;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+import org.neo4j.graphdb.GraphDatabaseService;
+import org.neo4j.test.TestGraphDatabaseFactory;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import static apoc.util.MapUtil.map;
+import static apoc.util.TestUtil.testCall;
+import static org.junit.Assert.assertEquals;
+
+/**
+ * @author Stefan Armbruster
+ */
+@RunWith(Parameterized.class)
+public class StringCleanTest {
+
+    private static GraphDatabaseService db;
+
+
+    @BeforeClass
+    public static void setUp() throws Exception {
+        db = new TestGraphDatabaseFactory().newImpermanentDatabase();
+        TestUtil.registerProcedure(db, Strings.class);
+    }
+
+    @AfterClass
+    public static void tearDown() {
+        db.shutdown();
+    }
+
+    @Parameters(name = "{0}")
+    public static Collection<Object[]> data() {
+        return Arrays.asList(new Object[][]{
+                { "&N[]eo  4 #J-(3.0)  ", "neo4j30"},
+                { "German umlaut Ä Ö Ü ä ö ü ß ", "germanumlautaeoeueaeoeuess" },
+                { "French çÇéèêëïîôœàâæùûü", "frenchcceeeeiioaauuue"}
+        });
+    }
+
+    @Parameter(value = 0)
+    public String dirty;
+
+    @Parameter(value = 1)
+    public String clean;
+
+    @Test
+    public void testClean() throws Exception {
+        testCall(db,
+                "CALL apoc.text.clean({a})",
+                map("a", dirty),
+                row -> assertEquals(clean, row.get("value")));
+    }
+
+}
diff --git a/src/test/java/apoc/text/StringsTest.java b/src/test/java/apoc/text/StringsTest.java
index a6e796505..062d3f64e 100644
--- a/src/test/java/apoc/text/StringsTest.java
+++ b/src/test/java/apoc/text/StringsTest.java
@@ -103,13 +103,6 @@ public class StringsTest {
                 row -> assertEquals(expected, row.get("value")));
     }
 
-    @Test public void testClean() throws Exception {
-        String testStringA = "&N[]eo  4 #J-(3.0)  ";
-        testCall(db,
-                "CALL apoc.text.clean({a})",
-                map("a",testStringA),
-                row -> assertEquals("neo4j30", row.get("value")));
-    }
 
     @Test public void testCleanWithNull() throws Exception {
         testCall(db,