Repository: mholt/PapaParse
Pull request #544 (2018-07-26 20:22:30)

New feature: Option to skip lines with no content
Added a boolean config value `skipNoContentLines` which skips lines that only contain delimiters, quotes, and whitespace, instead of just lines that evaluate to an empty string. Works if set alongside `skipEmptyLines` or on its own. Passing tests are included.

MonkeyDZeke: That's also an option. Whatever is most maintainable and user-friendly. We could use 'greedy', 'strict', 'no-content', or a few different options. Whatever we do, the documentation needs to be super clear and intuitive.
MonkeyDZeke: Having the mixed string and boolean input makes the demo for it a bit awkward so I removed that for now until we decide if we want to demo this particular feature and how it would be best done.
pokoli: For the demo I think we can preserve the current boolean flag and set keep the previous behaviour. 

Patch:
diff --git a/docs/demo.html b/docs/demo.html
index bdbfc8c..aa7cba9 100644
--- a/docs/demo.html
+++ b/docs/demo.html
@@ -89,6 +89,7 @@
 						<input type="checkbox" id="skipEmptyLines"> Skip empty lines
 						<dfn>By default, empty lines are parsed; check to skip.</dfn>
 					</label>
+
 				</div>
 
 				<div class="grid-75 grid-parent">
diff --git a/docs/docs.html b/docs/docs.html
index 37fe8f1..68f1132 100644
--- a/docs/docs.html
+++ b/docs/docs.html
@@ -505,7 +505,7 @@ var csv = Papa.unparse({
 									<code>skipEmptyLines</code>
 								</td>
 								<td>
-									If true, lines that are completely empty will be skipped. An empty line is defined to be one which evaluates to empty string.
+									If true, lines that are completely empty (those which evaluate to an empty string) will be skipped. If set to <code>'greedy'</code>, lines that don't have any content (those which have only whitespace after parsing) will also be skipped.
 								</td>
 							</tr>
 							<tr>
diff --git a/papaparse.js b/papaparse.js
index 0b3dc49..6d1bf63 100755
--- a/papaparse.js
+++ b/papaparse.js
@@ -1075,6 +1075,10 @@
 			_input = '';
 		};
 
+		function testEmptyLine(s) {
+			return _config.skipEmptyLines === 'greedy' ? s.join('').trim() === '' : s.length === 1 && s[0].length === 0;
+		}
+
 		function processResults()
 		{
 			if (_results && _delimiterError)
@@ -1086,7 +1090,7 @@
 			if (_config.skipEmptyLines)
 			{
 				for (var i = 0; i < _results.data.length; i++)
-					if (_results.data[i].length === 1 && _results.data[i][0] === '')
+					if (testEmptyLine(_results.data[i]))
 						_results.data.splice(i--, 1);
 			}
 
@@ -1215,7 +1219,8 @@
 
 				for (var j = 0; j < preview.data.length; j++)
 				{
-					if (skipEmptyLines && preview.data[j].length === 1 && preview.data[j][0].length === 0) {
+					if (skipEmptyLines && testEmptyLine(preview.data[j]))
+					{
 						emptyLinesCount++;
 						continue;
 					}
@@ -1291,7 +1296,8 @@
 	}
 
 	/** https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions */
-	function escapeRegExp(string) {
+	function escapeRegExp(string)
+	{
 		return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); // $& means the whole matched string
 	}
 


Test patch:
diff --git a/tests/test-cases.js b/tests/test-cases.js
index df5c152..be5cc5a 100644
--- a/tests/test-cases.js
+++ b/tests/test-cases.js
@@ -1338,6 +1338,26 @@ var PARSE_TESTS = [
 				truncated: false
 			}
 		}
+	},
+	{
+		description: "Parsing with skipEmptyLines set to 'greedy'",
+		notes: "Must parse correctly without lines with no content",
+		input: 'a,b\n\n,\nc,d\n , \n""," "\n	,	\n,,,,\n',
+		config: { skipEmptyLines: 'greedy' },
+		expected: {
+			data: [['a', 'b'], ['c', 'd']],
+			errors: []
+		}
+	},
+	{
+		description: "Parsing with skipEmptyLines set to 'greedy' with quotes and delimiters as content",
+		notes: "Must include lines with escaped delimiters and quotes",
+		input: 'a,b\n\n,\nc,d\n" , ",","\n""" """,""""""\n\n\n',
+		config: { skipEmptyLines: 'greedy' },
+		expected: {
+			data: [['a', 'b'], ['c', 'd'], [' , ', ','], ['" "', '""']],
+			errors: []
+		}
 	}
 ];