Repository: mholt/PapaParse (js)

Issue:
5.0.0-beta.0 header: true, worker: false, steps... incorrectly parses headers
Here is a failing test:
```
AssertionError: expected '[[{"O":"A"},{"O":"b"},{"O":"c"}],[{"O":"d"},{"O":"E"},{"O":"f"}]]' to deeply equal '[{"One":"A","Two":"b","Three":"c"},{"One":"d","Two":"E","Three":"f"}]'
      + expected - actual

      -[[{"O":"A"},{"O":"b"},{"O":"c"}],[{"O":"d"},{"O":"E"},{"O":"f"}]]
      +[{"One":"A","Two":"b","Three":"c"},{"One":"d","Two":"E","Three":"f"}]
```
<details>
<summary>code</summary>

```js
	{
		description: "Data is correctly parsed with steps (headers)",
		expected: [{One: 'A', Two: 'b', Three: 'c'}, {One: 'd', Two: 'E', Three: 'f'}],
		run: function(callback) {
			var data = [];
			Papa.parse('One,Two,Three\nA,b,c\nd,E,f', {
				header: true,
				step: function(results) {
					data.push(results.data);
				},
				complete: function() {
					callback(data);
				}
			});
		}
	},
	{
		// The worker code path does work,
		// This fails because node doesn't expose Workers yet ... so this falls back to the non-worker path.
		// See: https://nodejs.org/api/worker_threads.html
		description: "Data is correctly parsed with steps and worker (headers)",
		expected: [{One: 'A', Two: 'b', Three: 'c'}, {One: 'd', Two: 'E', Three: 'f'}],
		run: function(callback) {
			var data = [];
			Papa.parse('One,Two,Three\nA,b,c\nd,E,f', {
				header: true,
				worker: true,
				step: function(results) {
					data.push(results.data);
				},
				complete: function() {
					callback(data);
				}
			});
		}
	},
```

</details>

Pull request description:
Fixes #619
Adds tests and fixes an issue that incorrectly parses the headers with step functions

Patch:
diff --git a/docs/docs.html b/docs/docs.html
index 7680ef1..59c87d5 100644
--- a/docs/docs.html
+++ b/docs/docs.html
@@ -355,7 +355,8 @@ var csv = Papa.unparse({
 	fastMode: undefined,
 	beforeFirstChunk: undefined,
 	withCredentials: undefined,
-	transform: undefined
+	transform: undefined,
+	delimitersToGuess: [',', '\t', '|', ';', <a href="#readonly">Papa.RECORD_SEP</a>, <a href="#readonly">Papa.UNIT_SEP</a>]
 }</code></pre>
 					</div>
 					<div class="clear"></div>
@@ -375,7 +376,7 @@ var csv = Papa.unparse({
 									<code>delimiter</code>
 								</td>
 								<td>
-									The delimiting character. Leave blank to auto-detect from a list of most common delimiters. It can be a string or a function. If string, it must be one of length 1. If a function, it must accept the input as first parameter and it must return a string which will be used as delimiter. In both cases it cannot be found in <a href="#readonly">Papa.BAD_DELIMITERS</a>.
+									The delimiting character. Leave blank to auto-detect from a list of most common delimiters, or any values passed in through <code>delimitersToGuess</code>. It can be a string or a function. If string, it must be one of length 1. If a function, it must accept the input as first parameter and it must return a string which will be used as delimiter. In both cases it cannot be found in <a href="#readonly">Papa.BAD_DELIMITERS</a>.
 								</td>
 							</tr>
 							<tr>
@@ -415,7 +416,8 @@ var csv = Papa.unparse({
 									<code>transformHeader</code>
 								</td>
 								<td>
-									A function to apply on each header. Requires <code>header</code> to be <code>true</code>. The function receives the header as its first argument.
+									A function to apply on each header. Requires <code>header</code> to be <code>true</code>. The function receives the header as its first argument.<br>
+									Only available starting with version 5.0.
 								</td>
 							</tr>
 							<tr>
@@ -548,6 +550,14 @@ var csv = Papa.unparse({
 									A function to apply on each value. The function receives the value as its first argument and the column number or header name when enabled as its second argument. The return value of the function will replace the value it received. The transform function is applied before dynamicTyping.
 								</td>
 							</tr>
+							<tr>
+								<td>
+									<code>delimitersToGuess</code>
+								</td>
+								<td>
+									An array of delimiters to guess from if the <code>delimiter</code> option is not set.
+								</td>
+							</tr>
 						</table>
 					</div>
 				</div>
diff --git a/papaparse.js b/papaparse.js
index 6f18efe..5ed703e 100755
--- a/papaparse.js
+++ b/papaparse.js
@@ -1031,7 +1031,7 @@ License: MIT
 			_delimiterError = false;
 			if (!_config.delimiter)
 			{
-				var delimGuess = guessDelimiter(input, _config.newline, _config.skipEmptyLines, _config.comments);
+				var delimGuess = guessDelimiter(input, _config.newline, _config.skipEmptyLines, _config.comments, _config.delimitersToGuess);
 				if (delimGuess.successful)
 					_config.delimiter = delimGuess.bestDelimiter;
 				else
@@ -1125,18 +1125,25 @@ License: MIT
 		{
 			if (!_results)
 				return;
-			for (var i = 0; needsHeaderRow() && i < _results.data.length; i++)
-				for (var j = 0; j < _results.data[i].length; j++)
-				{
-					var header = _results.data[i][j];
 
-					if (isFunction(_config.transformHeader)) {
-						header = _config.transformHeader(header);
-					}
+			function addHeder(header)
+			{
+				if (isFunction(_config.transformHeader))
+					header = _config.transformHeader(header);
 
-					_fields.push(header);
-				}
-			_results.data.splice(0, 1);
+				_fields.push(header);
+			}
+
+			if (Array.isArray(_results.data[0]))
+			{
+				for (var i = 0; needsHeaderRow() && i < _results.data.length; i++)
+					_results.data[i].forEach(addHeder);
+
+				_results.data.splice(0, 1);
+			}
+			// if _results.data[0] is not an array, we are in a step where _results.data is the row.
+			else
+				_results.data.forEach(addHeder);
 		}
 
 		function shouldApplyDynamicTyping(field) {
@@ -1170,15 +1177,15 @@ License: MIT
 			if (!_results || (!_config.header && !_config.dynamicTyping && !_config.transform))
 				return _results;
 
-			for (var i = 0; i < _results.data.length; i++)
+			function processRow(rowSource, i)
 			{
 				var row = _config.header ? {} : [];
 
 				var j;
-				for (j = 0; j < _results.data[i].length; j++)
+				for (j = 0; j < rowSource.length; j++)
 				{
 					var field = j;
-					var value = _results.data[i][j];
+					var value = rowSource[j];
 
 					if (_config.header)
 						field = j >= _fields.length ? '__parsed_extra' : _fields[j];
@@ -1197,7 +1204,6 @@ License: MIT
 						row[field] = value;
 				}
 
-				_results.data[i] = row;
 
 				if (_config.header)
 				{
@@ -1206,23 +1212,36 @@ License: MIT
 					else if (j < _fields.length)
 						addError('FieldMismatch', 'TooFewFields', 'Too few fields: expected ' + _fields.length + ' fields but parsed ' + j, _rowCounter + i);
 				}
+
+				return row;
+			}
+
+			var incrementBy = 1;
+			if (!_results.data[0] || Array.isArray(_results.data[0]))
+			{
+				_results.data = _results.data.map(processRow);
+				incrementBy = _results.data.length;
 			}
+			else
+				_results.data = processRow(_results.data, 0);
+
 
 			if (_config.header && _results.meta)
 				_results.meta.fields = _fields;
 
-			_rowCounter += _results.data.length;
+			_rowCounter += incrementBy;
 			return _results;
 		}
 
-		function guessDelimiter(input, newline, skipEmptyLines, comments)
+		function guessDelimiter(input, newline, skipEmptyLines, comments, delimitersToGuess)
 		{
-			var delimChoices = [',', '\t', '|', ';', Papa.RECORD_SEP, Papa.UNIT_SEP];
 			var bestDelim, bestDelta, fieldCountPrevRow;
 
-			for (var i = 0; i < delimChoices.length; i++)
+			delimitersToGuess = delimitersToGuess || [',', '\t', '|', ';', Papa.RECORD_SEP, Papa.UNIT_SEP];
+
+			for (var i = 0; i < delimitersToGuess.length; i++)
 			{
-				var delim = delimChoices[i];
+				var delim = delimitersToGuess[i];
 				var delta = 0, avgFieldCount = 0, emptyLinesCount = 0;
 				fieldCountPrevRow = undefined;
 


Test patch:
diff --git a/tests/test-cases.js b/tests/test-cases.js
index e8b424f..b0610dd 100644
--- a/tests/test-cases.js
+++ b/tests/test-cases.js
@@ -1804,6 +1804,39 @@ var CUSTOM_TESTS = [
 			});
 		}
 	},
+	{
+		description: "Data is correctly parsed with steps (headers)",
+		expected: [{One: 'A', Two: 'b', Three: 'c'}, {One: 'd', Two: 'E', Three: 'f'}],
+		run: function(callback) {
+			var data = [];
+			Papa.parse('One,Two,Three\nA,b,c\nd,E,f', {
+				header: true,
+				step: function(results) {
+					data.push(results.data);
+				},
+				complete: function() {
+					callback(data);
+				}
+			});
+		}
+	},
+	{
+		description: "Data is correctly parsed with steps and worker (headers)",
+		expected: [{One: 'A', Two: 'b', Three: 'c'}, {One: 'd', Two: 'E', Three: 'f'}],
+		run: function(callback) {
+			var data = [];
+			Papa.parse('One,Two,Three\nA,b,c\nd,E,f', {
+				header: true,
+				worker: true,
+				step: function(results) {
+					data.push(results.data);
+				},
+				complete: function() {
+					callback(data);
+				}
+			});
+		}
+	},
 	{
 		description: "Data is correctly parsed with steps and worker",
 		expected: [['A', 'b', 'c'], ['d', 'E', 'f']],
@@ -2187,8 +2220,25 @@ var CUSTOM_TESTS = [
 				}
 			});
 		}
+	},
+	{
+		description: "Should correctly guess custom delimiter when passed delimiters to guess.",
+		expected: "~",
+		run: function(callback) {
+			var results = Papa.parse('"A"~"B"~"C"~"D"', {
+				delimitersToGuess: ['~', '@', '%']
+			});
+			callback(results.meta.delimiter);
+		}
+	},
+	{
+		description: "Should still correctly guess default delimiters when delimiters to guess are not given.",
+		expected: ",",
+		run: function(callback) {
+			var results = Papa.parse('"A","B","C","D"');
+			callback(results.meta.delimiter);
+		}
 	}
-
 ];
 
 describe('Custom Tests', function() {