Repository: mholt/PapaParse
Pull request #1021 (2023-09-24 15:41:29)

Skip lines future (Feature Suggestion) #738
 
## Issues No : #738

### Description:
In this feature enhancement, Added the capability to skip a defined number of lines when converting a CSV (Comma-Separated Values) file to JSON format. This functionality offers users more flexibility when dealing with CSV files with header information or metadata at the beginning, allowing for a more efficient and precise conversion process.

### Changes Made:

1. Introduced a new configuration parameter, skipFirstNLines, which allows users to specify the number of lines to skip at the beginning of the CSV file during the conversion to JSON. This parameter supports values from 0 and above.
2. Implemented validation for the skipFirstNLines configuration parameter to ensure that it does not negatively impact the parsing process, even if it is set to 0 or given a negative value.
3. Enhanced the user interface of the player.html file by adding an input field that allows users to input the desired value for skipFirstNLines.
4.  Updated the player.js script to retrieve and utilize the skipFirstNLines input from the user and incorporate it into the parser's configuration.


### Test cases:
 
Included test cases to validate the functionality of the skipFirstNLines feature, ensuring that it behaves correctly under various scenarios.

Test 1:  "Skip First N number of lines , with header",

Test 2: "Skip First N number of lines , without header",

Test 3: "Skip  First N lines with value as 0 with header false",

Test 4:  "Skip First N lines with value as 0 with header true",

Test 5: "Skip First N lines with value less then 0, with header true",
		
Test 6: "Without Skip First N Lines config",


pokoli: What you are proposing is already implemented using the `preview` config option. 
bhuvaneshwararaja: Hi @pokoli , 
I have gone through the documentation (https://www.papaparse.com/docs) for the  preview options 
and got to know that will help to parse the expected number of  rows in csv
example: If we have 20 rows in csv, If the preview value is 10 . Out of 20 only 10 rows will be parsed . 

But this PR will add a config to skip the N number of rows in the beginning of the csv.
Example: If we have 20 rows in csv , If skipNLines value is 10 . Out of 20 , first 10 rows will be skipped .

Please check it once again , it is relevant to the issue #738 as well. 

pokoli: Ok, then we need to document this new flag to explain which is the ussage. 

I think the flag will be better named as `skipFirstLines`. 

Could you have a look at the test suite? it seems you introduced some failing tests.
bhuvaneshwararaja: Hi @pokoli , Thanks 
Sure i will check those test suite , I will update the PR with doc
bhuvaneshwararaja: Hi @pokoli , I have updated the PR 


Patch:
diff --git a/docs/docs.html b/docs/docs.html
index 49095b1..b9292fa 100644
--- a/docs/docs.html
+++ b/docs/docs.html
@@ -450,7 +450,8 @@ var csv = Papa.unparse({
 	beforeFirstChunk: undefined,
 	withCredentials: undefined,
 	transform: undefined,
-	delimitersToGuess: [',', '\t', '|', ';', <a href="#readonly">Papa.RECORD_SEP</a>, <a href="#readonly">Papa.UNIT_SEP</a>]
+	delimitersToGuess: [',', '\t', '|', ';', <a href="#readonly">Papa.RECORD_SEP</a>, <a href="#readonly">Papa.UNIT_SEP</a>],
+	skipFirstNLines: 0
 }</code></pre>
 					</div>
 					<div class="clear"></div>
@@ -682,6 +683,14 @@ var csv = Papa.unparse({
 									An array of delimiters to guess from if the <code>delimiter</code> option is not set.
 								</td>
 							</tr>
+							<tr>
+								<td>
+									<code>skipFirstNLines</code>
+								</td>
+								<td>
+									To skip first N number of lines when converting a CSV file to JSON 
+								</td>
+							</tr>
 						</table>
 					</div>
 				</div>
diff --git a/papaparse.js b/papaparse.js
index 3ce3962..4bc3dc3 100755
--- a/papaparse.js
+++ b/papaparse.js
@@ -377,8 +377,10 @@ License: MIT
 				_escapedQuote = _config.escapeChar + _quoteChar;
 			}
 
-			if (typeof _config.escapeFormulae === 'boolean' || _config.escapeFormulae instanceof RegExp) {
-				_escapeFormulae = _config.escapeFormulae instanceof RegExp ? _config.escapeFormulae : /^[=+\-@\t\r].*$/;
+			if (_config.escapeFormulae instanceof RegExp) {
+				_escapeFormulae = _config.escapeFormulae;
+			} else if (typeof _config.escapeFormulae === 'boolean' && _config.escapeFormulae) {
+				_escapeFormulae =  /^[=+\-@\t\r].*$/;
 			}
 		}
 
@@ -484,6 +486,7 @@ License: MIT
 		}
 	}
 
+
 	/** ChunkStreamer is the base prototype for various streamer implementations. */
 	function ChunkStreamer(config)
 	{
@@ -519,8 +522,23 @@ License: MIT
 
 			// Rejoin the line we likely just split in two by chunking the file
 			var aggregate = this._partialLine + chunk;
+			this._pendingSkip = parseInt(this._config.skipFirstNLines) || 0;
+			this._skipHeader = 0;
+			if (this._config.header) {
+				this._skipHeader++;
+			}
+			if (this._pendingSkip > 0) {
+				var splitChunk = aggregate.split('\n');
+				var currentChunkLength = splitChunk.length;
+				if (currentChunkLength <= this._pendingSkip) {
+					aggregate = this._partialLine;
+				}
+				else{
+					aggregate = this._partialLine + [...splitChunk.slice(0, this._skipHeader), ...splitChunk.slice(this._skipHeader + this._pendingSkip)].join('\n');
+				}
+				this._pendingSkip -= currentChunkLength;
+			}
 			this._partialLine = '';
-
 			var results = this._handle.parse(aggregate, this._baseIndex, !this._finished);
 
 			if (this._handle.paused() || this._handle.aborted()) {
@@ -1929,7 +1947,6 @@ License: MIT
 	{
 		return function() { f.apply(self, arguments); };
 	}
-
 	function isFunction(func)
 	{
 		return typeof func === 'function';
diff --git a/player/player.html b/player/player.html
index b90fe98..48015e3 100644
--- a/player/player.html
+++ b/player/player.html
@@ -24,6 +24,7 @@
 				<label><input type="checkbox" id="skipEmptyLines"> Skip empty lines</label>
 				<label><input type="checkbox" id="step-pause"> Pause on step</label>
 				<label><input type="checkbox" id="print-steps"> Log each step/chunk</label>
+				<label>Skip First N Lines: <input type="number" id="skipFirstNLines"></label>
 				
 				<label>Delimiter: <input type="text" size="5" placeholder="auto" id="delimiter"> <a href="javascript:" id="insert-tab">tab</a></label>
 
diff --git a/player/player.js b/player/player.js
index 8150de6..f8b8e3a 100644
--- a/player/player.js
+++ b/player/player.js
@@ -108,6 +108,7 @@ function buildConfig()
 		skipEmptyLines: $('#skipEmptyLines').prop('checked'),
 		chunk: $('#chunk').prop('checked') ? chunkFn : undefined,
 		beforeFirstChunk: undefined,
+		skipFirstNLines: $('#skipFirstNLines').val()
 	};
 
 	function getLineEnding()


Test patch:
diff --git a/tests/test-cases.js b/tests/test-cases.js
index da489e8..68f367f 100644
--- a/tests/test-cases.js
+++ b/tests/test-cases.js
@@ -1574,6 +1574,42 @@ var PARSE_TESTS = [
 			data: [['a', 'b', 'c\n'], ['d', 'e', 'f']],
 			errors: []
 		}
+	},
+	{
+		description: "Skip First N number of lines , with header and 2 rows",
+		input: 'a,b,c,d\n1,2,3,4',
+		config: { header: true, skipFirstNLines: 1 },
+		expected: {
+			data: [],
+			errors: []
+		}
+	},
+	{
+		description: "Skip First N number of lines , with header and 3 rows",
+		input: 'a,b,c,d\n1,2,3,4\n4,5,6,7',
+		config: { header: true, skipFirstNLines: 1 },
+		expected: {
+			data: [{a: '4', b: '5', c: '6', d: '7'}],
+			errors: []
+		}
+	},
+	{
+		description: "Skip First N number of lines , with header false",
+		input: 'a,b,c,d\n1,2,3,4\n4,5,6,7',
+		config: { header: false, skipFirstNLines: 1 },
+		expected: {
+			data: [['1','2','3','4'],['4','5','6','7']],
+			errors: []
+		}
+	},
+	{
+		description: "Skip First N number of lines , with header false and skipFirstNLines as negative value",
+		input: 'a,b,c,d\n1,2,3,4\n4,5,6,7',
+		config: { header: false, skipFirstNLines: -2 },
+		expected: {
+			data: [['a','b','c','d'],['1','2','3','4'],['4','5','6','7']],
+			errors: []
+		}
 	}
 ];