Repository: mholt/PapaParse
Pull request #460 (2018-03-05 20:19:33)

Add option to configure escape character (#459)
This pull request adds the optional configuration option `escapeChar` to escape the `quoteChar` in a quoted field. This option allows to parse CSV files where a non-standard escape character is used. (p.E. `\"` instead of `""`) .

If `escapeChar` is provided, it may contain a one character long string.
If `escapeChar` is not provided, the `quoteChar` will be used as fallback.
The `escapeChar` must appear before the `quoteChar`.

Two tests have been added to validate the functionality of this feature.
Solves issue #459.

pokoli: Hi, thanks for the pull request and sorry for the delay. I've been bussy and I haven't been able to review until now. 

I'm wondering what will happen if the text contains the escapeChar. For example including a field with : _"\\a"_, should be parsed as _"\a"_

Could you please add a test case that tests this behaviour also? 

It will be great if you create also a PR on the gh-pages branch to update the documentation including this new parameter. 
marius-wieschollek: I added the test and made a pull request for the documentation.

Usually the escaped quote character is detected by searching for the next occurrence of the quoteChar and then checking if the next character is the quoteChar and if so skipping both.
When a custom escapeChar is used and the quoteChar is found, i check if the previous character is the escapeChar, not the next one, and if so only the current character is skipped. This should by design already prevent all other occurrences of the escapeChar from being altered.

Patch:
diff --git a/papaparse.js b/papaparse.js
index c1f1ec4..048bd46 100755
--- a/papaparse.js
+++ b/papaparse.js
@@ -1158,6 +1158,10 @@
 		} else {
 			var quoteChar = config.quoteChar;
 		}
+		var escapeChar = quoteChar;
+		if (config.escapeChar !== undefined){
+			escapeChar = config.escapeChar;
+		}
 
 		// Delimiter must be valid
 		if (typeof delim !== 'string'
@@ -1236,7 +1240,7 @@
 
 			var nextDelim = input.indexOf(delim, cursor);
 			var nextNewline = input.indexOf(newline, cursor);
-			var quoteCharRegex = new RegExp(quoteChar+quoteChar, 'g');
+			var quoteCharRegex = new RegExp(escapeChar.replace(/[\-\[\]\/\{\}\(\)\*\+\?\.\\\^\$\|]/g, '\\$&')+quoteChar, 'g');
 
 			// Parser loop
 			for (;;)
@@ -1279,12 +1283,19 @@
 						}
 
 						// If this quote is escaped, it's part of the data; skip it
-						if (input[quoteSearch+1] === quoteChar)
+						// If the quote character is the escape character, then check if the next character is the escape character
+						if (quoteChar === escapeChar &&  input[quoteSearch+1] === escapeChar)
 						{
 							quoteSearch++;
 							continue;
 						}
 
+						// If the quote character is not the escape character, then check if the previous character was the escape character
+						if (quoteChar !== escapeChar && quoteSearch !== 0 && input[quoteSearch-1] === escapeChar)
+						{
+							continue;
+						}
+
 						var spacesBetweenQuoteAndDelimiter = extraSpaces(nextDelim);
 
 						// Closing quote followed by delimiter or 'unnecessary steps + delimiter'


Test patch:
diff --git a/tests/test-cases.js b/tests/test-cases.js
index 503e905..ea7fdcc 100644
--- a/tests/test-cases.js
+++ b/tests/test-cases.js
@@ -1080,6 +1080,36 @@ var PARSE_TESTS = [
 			errors: []
 		}
 	},
+	{
+		description: "Custom escape character in the middle",
+		notes: "Must parse correctly if the backslash sign (\\) is configured as a custom escape character",
+		input: 'a,b,"c\\"d\\"f"',
+		config: { escapeChar:'\\'},
+		expected: {
+			data: [['a', 'b', 'c"d"f']],
+			errors: []
+		}
+	},
+	{
+		description: "Custom escape character at the end",
+		notes: "Must parse correctly if the backslash sign (\\) is configured as a custom escape character and the escaped quote character appears at the end of the column",
+		input: 'a,b,"c\\"d\\""',
+		config: { escapeChar:'\\'},
+		expected: {
+			data: [['a', 'b', 'c"d"']],
+			errors: []
+		}
+	},
+	{
+		description: "Custom escape character not used for escaping",
+		notes: "Must parse correctly if the backslash sign (\\) is configured as a custom escape character and appears as regular character in the text",
+		input: 'a,b,"c\\d"',
+		config: { escapeChar:'\\'},
+		expected: {
+			data: [['a', 'b', 'c\\d']],
+			errors: []
+		}
+	},
 	{
 		description: "Header row with preceding comment",
 		notes: "Must parse correctly headers if they are preceded by comments",