Repository: Bridgeconn/usfm-grammar (language: python)

Issue:
Replicate python parser in javascript
The usfm-grammar implementation is now done in python, in the module `python-usfm-parser`. Replicate the same functionalities into javascript and make the `javascript-usfm-parser` module


Discussion / hints:

Fix patch:
diff --git a/node-usfm-parser/src/listGenerator.js b/node-usfm-parser/src/listGenerator.js
new file mode 100644
index 00000000..e5d238f7
--- /dev/null
+++ b/node-usfm-parser/src/listGenerator.js
@@ -0,0 +1,56 @@
+class ListGenerator {
+    /* Combines the methods used for List generation from USJ */
+    constructor() {
+        /* Variables shared by functions */
+        this.book = "";
+        this.currentChapter = "";
+        this.currentVerse = "";
+        this.list = [["Book", "Chapter", "Verse", "Text", "Type", "Marker"]];
+    }
+
+    usjToListId(obj) {
+        /* Update book code */
+        this.book = obj.code;
+    }
+
+    usjToListC(obj) {
+        /* Update current chapter */
+        this.currentChapter = obj.number;
+    }
+
+    usjToListV(obj) {
+        /* Update current verse */
+        this.currentVerse = obj.number;
+    }
+
+    usjToList(obj) {
+        /* Traverse the USJ dict and build the table in this.list */
+        if (obj.type === "book") {
+            this.usjToListId(obj);
+        } else if (obj.type === "chapter") {
+            this.usjToListC(obj);
+        } else if (obj.type === "verse") {
+            this.usjToListV(obj);
+        }
+
+        let markerType = obj.type;
+        let markerName = obj.marker ? obj.marker : '';
+        
+        if (markerType === "USJ") {
+            // This would occur if the JSON got flattened after removing paragraph markers
+            markerType = "";
+        }
+
+        if (obj.content) {
+            for (let item of obj.content) {
+                if (typeof item === "string") {
+                    this.list.push([this.book, this.currentChapter, this.currentVerse, item, markerType, markerName]);
+                } else {
+                    this.usjToList(item);
+                }
+            }
+        }
+    }
+}
+
+exports.ListGenerator = ListGenerator;
diff --git a/node-usfm-parser/src/usfmParser.js b/node-usfm-parser/src/usfmParser.js
index 93f69463..94bf26c7 100644
--- a/node-usfm-parser/src/usfmParser.js
+++ b/node-usfm-parser/src/usfmParser.js
@@ -2,6 +2,7 @@ const Parser = require('tree-sitter');
 
 const {USFMGenerator} = require("./usfmGenerator");
 const {USJGenerator} = require("./usjGenerator"); 
+const {ListGenerator} = require("./listGenerator");
 const { includeMarkersInUsj, excludeMarkersInUsj, Filter } = require("./filters.js");
 const USFM3 = require('tree-sitter-usfm3');
 const { Query } = Parser;
@@ -176,20 +177,42 @@ Only one of USFM, USJ or USX is supported in one object.`)
 
 		return outputUSJ;
 	}
-}
 
+	toList(
+	    excludeMarkers = null,
+	    includeMarkers = null,
+	    ignoreErrors = false,
+	    combineTexts = true
+	) {
+	    /* Uses the toJSON function and converts JSON to CSV
+	       To be re-implemented to work with the flat JSON schema */
+
+	    if (!ignoreErrors && this.errors && this.errors.length > 0) {
+	        const errStr = this.errors.map(err => err.join(":")).join("\n\t");
+	        throw new Error(`Errors present:\n\t${errStr}\nUse ignoreErrors=true to generate output despite errors`);
+	    }
+
+	    try {
+	        const usjDict = this.toUSJ(excludeMarkers, includeMarkers, ignoreErrors, combineTexts);
+
+	        const listGenerator = new ListGenerator();
+	        listGenerator.usjToList(usjDict);
+	    	return listGenerator.list;
 
+	    } catch (exe) {
+	        let message = "Unable to do the conversion. ";
+	        if (this.errors && this.errors.length > 0) {
+	            const errStr = this.errors.map(err => err.join(":")).join("\n\t");
+	            message += `Could be due to an error in the USFM\n\t${errStr}`;
+	        }
+	        throw new Error(message, { cause: exe });
+	    }
+
+	}
 
-class Format {
-  // Defines the valid values for input and output formats
-  static JSON = "usj";
-  static CSV = "table";
-  static ST = "syntax-tree";
-  static USX = "usx";
-  static MD = "markdown";
-  static USFM = "usfm";
 }
 
+
 exports.USFMParser = USFMParser;
 exports.Filter = Filter;
-exports.Format = Format;
\ No newline at end of file
+// exports.Format = Format;
\ No newline at end of file
diff --git a/node-usfm-parser/src/usfmParserWeb.js b/node-usfm-parser/src/usfmParserWeb.js
deleted file mode 100644
index e4e35dcf..00000000
--- a/node-usfm-parser/src/usfmParserWeb.js
+++ /dev/null
@@ -1,143 +0,0 @@
-const Parser = require('web-tree-sitter');
-
-const {USFMGenerator} = require("./usfmGenerator");
-const {USJGenerator} = require("./usjGenerator"); 
-const { includeMarkersInUsj, excludeMarkersInUsj } = require("./filters.js");
-
-let USFM3 = null;
-
-class USFMParser {
-
-	constructor() {
-		this.parser = null;
-		this.usfm = null;
-		this.usj = null;
-		this.syntaxTree = null;
-		this.language = null;
-		this.errorQuery = null;
-		this.initializeParser()
-	}
-
-	static async init(grammarPath='src/grammar/tree-sitter-usfm.wasm') {
-		await Parser.init();
-		USFM3 = await Parser.Language.load(grammarPath);
-	}
-
-	initializeParser() {
-		this.parser = new Parser();
-		this.language = USFM3;
-		this.parser.setLanguage(USFM3);
-
-
-	}
-
-	usfmToUsj(usfmString) {
-		if (typeof usfmString !== "string" || usfmString === null) {
-			throw new Error("Invalid input for USFM. Expected a string.");
-		}
-		if (!this.parser) {
-			this.init();
-		}
-		this.usfm = usfmString;
-		this.parseUSFM();
-		this.usj = this.convertUSFMToUSJ();
-		return this.usj;
-	}
-
-	usjToUsfm(usjObject) {
-		if (typeof usjObject !== "object" || usjObject === null) {
-			throw new Error("Invalid input for USJ. Expected an object.");
-		}
-		if (!this.parser) {
-			this.initializeParser();
-		}
-		this.usj = usjObject;
-		this.usfm = this.convertUSJToUSFM();
-		return this.usfm;
-	}
-
-	parseUSFM() {
-		let tree = null;
-		try {
-			tree = this.parser.parse(this.usfm);
-		} catch (err) {
-			console.log(err.toString());
-		}
-		const error = this.checkForErrors(tree);
-		if (error) throw error;
-		this.syntaxTree = tree.rootNode;
-	}
-
-
-	checkForErrors(tree) {
-		const errorQuery = this.parser.getLanguage().query("(ERROR) @errors");
-		const errors = errorQuery.captures(tree.rootNode);
-		if (errors.length > 0) {
-			this.errors = errors.map(
-				(err) =>
-					`At ${err.node.startPosition.row}:${err.node.startPosition.column}, Error: ${this.usfm.substring(err.node.startIndex, err.node.endIndex)}`,
-			);
-			return new Error(`Errors found in USFM: ${this.errors.join(", ")}`);
-		}
-		return null;
-	}
-
-	convertUSJToUSFM() {
-		const outputUSFM = new USFMGenerator().usjToUsfm(this.usj); // Simulated conversion
-		return outputUSFM;
-	}
-
-	convertUSFMToUSJ({
-		excludeMarkers = null,
-		includeMarkers = null,
-		ignoreErrors = false,
-		combineTexts = true,
-	} = {}) {
-		if (!ignoreErrors && this.errors) {
-			let errorString = this.errors.map((err) => err.join(":")).join("\n\t");
-			throw new Error(
-				`Errors present:\n\t${errorString}\nUse ignoreErrors = true to generate output despite errors.`,
-			);
-		}
-
-		let outputUSJ;
-		try {
-			let usjGenerator = new USJGenerator(
-				this.language,
-				this.usfm,
-				this.queryClass
-			);
-
-			usjGenerator.nodeToUSJ(this.syntaxTree, usjGenerator.jsonRootObj);
-			outputUSJ = usjGenerator.jsonRootObj;
-		} catch (err) {
-			let message = "Unable to do the conversion. ";
-			if (this.errors) {
-				let errorString = this.errors.map((err) => err.join(":")).join("\n\t");
-				message += `Could be due to an error in the USFM\n\t${errorString}`;
-			}
-			else {
-				message = err.message;
-			}
-			return {error: message};
-		}
-
-		if (includeMarkers) {
-			outputUSJ = includeMarkersInUsj(outputUSJ, [...includeMarkers, 'USJ'], combineTexts);
-		}
-		if (excludeMarkers) {
-			outputUSJ = excludeMarkersInUsj(outputUSJ, excludeMarkers, combineTexts);
-		}
-
-		return outputUSJ;
-	}
-}
-
-async function test() {
-	myparser = new USFMParser();
-	await myparser.init();
-	console.log(myparser.usfmToUsj('\\id GEN\n\\c 1\n\\p\n\\v 1 test text'));
-}
-
-// test();
-exports.USFMParser = USFMParser;
diff --git a/web-usfm-parser/src/listGenerator.js b/web-usfm-parser/src/listGenerator.js
new file mode 100644
index 00000000..a7e10b43
--- /dev/null
+++ b/web-usfm-parser/src/listGenerator.js
@@ -0,0 +1,56 @@
+class ListGenerator {
+    /* Combines the methods used for List generation from USJ */
+    constructor() {
+        /* Variables shared by functions */
+        this.book = "";
+        this.currentChapter = "";
+        this.currentVerse = "";
+        this.list = [["Book", "Chapter", "Verse", "Text", "Type", "Marker"]];
+    }
+
+    usjToListId(obj) {
+        /* Update book code */
+        this.book = obj.code;
+    }
+
+    usjToListC(obj) {
+        /* Update current chapter */
+        this.currentChapter = obj.number;
+    }
+
+    usjToListV(obj) {
+        /* Update current verse */
+        this.currentVerse = obj.number;
+    }
+
+    usjToList(obj) {
+        /* Traverse the USJ dict and build the table in this.list */
+        if (obj.type === "book") {
+            this.usjToListId(obj);
+        } else if (obj.type === "chapter") {
+            this.usjToListC(obj);
+        } else if (obj.type === "verse") {
+            this.usjToListV(obj);
+        }
+
+        let markerType = obj.type;
+        let markerName = obj.marker ? obj.marker : '';
+        
+        if (markerType === "USJ") {
+            // This would occur if the JSON got flattened after removing paragraph markers
+            markerType = "";
+        }
+
+        if (obj.content) {
+            for (let item of obj.content) {
+                if (typeof item === "string") {
+                    this.list.push([this.book, this.currentChapter, this.currentVerse, item, markerType, markerName]);
+                } else {
+                    this.usjToList(item);
+                }
+            }
+        }
+    }
+}
+
+export default ListGenerator;
diff --git a/web-usfm-parser/src/usfmParser.js b/web-usfm-parser/src/usfmParser.js
index 837e7d78..9e8ffd51 100644
--- a/web-usfm-parser/src/usfmParser.js
+++ b/web-usfm-parser/src/usfmParser.js
@@ -2,6 +2,7 @@ import Parser from './web-tree-sitter/tree-sitter.js';
 
 import USFMGenerator from "./usfmGenerator.js";
 import USJGenerator from "./usjGenerator.js";
+import ListGenerator from "./listGenerator.js"
 import { Filter } from "./filters.js";
 
 
@@ -191,6 +192,39 @@ Only one of USFM, USJ or USX is supported in one object.`)
 
 		return outputUSJ;
 	}
+
+	toList(
+	    excludeMarkers = null,
+	    includeMarkers = null,
+	    ignoreErrors = false,
+	    combineTexts = true
+	) {
+	    /* Uses the toJSON function and converts JSON to CSV
+	       To be re-implemented to work with the flat JSON schema */
+
+	    if (!ignoreErrors && this.errors && this.errors.length > 0) {
+	        const errStr = this.errors.map(err => err.join(":")).join("\n\t");
+	        throw new Error(`Errors present:\n\t${errStr}\nUse ignoreErrors=true to generate output despite errors`);
+	    }
+
+	    try {
+	        const usjDict = this.toUSJ(excludeMarkers, includeMarkers, ignoreErrors, combineTexts);
+
+	        const listGenerator = new ListGenerator();
+	        listGenerator.usjToList(usjDict);
+	    	return listGenerator.list;
+
+	    } catch (exe) {
+	        let message = "Unable to do the conversion. ";
+	        if (this.errors && this.errors.length > 0) {
+	            const errStr = this.errors.map(err => err.join(":")).join("\n\t");
+	            message += `Could be due to an error in the USFM\n\t${errStr}`;
+	        }
+	        throw new Error(message, { cause: exe });
+	    }
+
+	}
+
 }
 
 export {USFMParser, Filter};


Test patch:
diff --git a/node-usfm-parser/test/test_list_conversion.js b/node-usfm-parser/test/test_list_conversion.js
new file mode 100644
index 00000000..e16f087a
--- /dev/null
+++ b/node-usfm-parser/test/test_list_conversion.js
@@ -0,0 +1,71 @@
+const assert = require('assert');
+const fs = require('node:fs');
+const {allUsfmFiles, initialiseParser, isValidUsfm, excludeUSJs, findAllMarkers} = require('./config');
+const {USFMParser, Filter} = require("../src/index");
+
+
+describe("Check successful USFM-List conversion for positive samples", () => {
+
+  allUsfmFiles.forEach(function(value) {
+    if (isValidUsfm[value]) {
+      it(`Convert ${value} to List`, (inputUsfmPath=value) => {
+        //Tests if input parses without errors
+        const testParser = initialiseParser(inputUsfmPath)
+        assert(testParser instanceof USFMParser)
+        const list = testParser.toList();
+        assert(list instanceof Array);
+        assert.deepStrictEqual(list[0],
+        			[ 'Book', 'Chapter', 'Verse', 'Text', 'Type', 'Marker' ]); 
+
+      });
+    }
+  });
+});
+
+
+describe("Test Exclude Marker option in List conversion", () => {
+    // Test Exclude Maker option by checking markers in the List
+    const excludeTests = [
+            ['s', 'r']
+            ]
+    excludeTests.forEach(function(exList) {
+        allUsfmFiles.forEach(function(value) {
+          if (isValidUsfm[value]) {
+            it(`Exclude ${exList.slice(0, 5)} from ${value}`, (inputUsfmPath=value) => {
+                const testParser = initialiseParser(inputUsfmPath)
+                assert(testParser instanceof USFMParser)
+                const list = testParser.toList(excludeMarkers=exList);
+                assert(list instanceof Array);
+
+                const allTypes = list.map(row => row[5]);
+                let types = new Set(allTypes);
+                let intersection = exList.filter(value => types.has(value));
+                assert.deepStrictEqual(intersection, [])
+            });
+          }
+        })
+    })
+});
+
+describe("Test include Marker option in List conversion", () => {
+    // Test include Maker option by checking markers in the List
+    const includeTests = [
+            ['id', 'c', 'v']+Filter.TEXT+Filter.PARAGRAPHS
+            ]
+    includeTests.forEach(function(inList) {
+        allUsfmFiles.forEach(function(value) {
+          if (isValidUsfm[value]) {
+            it(`include ${inList.slice(0, 5)} of ${value} in List`, (inputUsfmPath=value) => {
+                const testParser = initialiseParser(inputUsfmPath)
+                assert(testParser instanceof USFMParser)
+                const list = testParser.toList(null, inList);
+                assert(list instanceof Array);
+
+                const allTypes = list.slice(1).map(row => row[5]);
+                assert( allTypes.every(element => inList.includes(element)), allTypes)
+
+            });
+          }
+        })
+    })
+});
\ No newline at end of file
diff --git a/node-usfm-parser/test/test_usj_conversion.js b/node-usfm-parser/test/test_usj_conversion.js
index b3f18cd4..2c1107a0 100644
--- a/node-usfm-parser/test/test_usj_conversion.js
+++ b/node-usfm-parser/test/test_usj_conversion.js
@@ -13,7 +13,7 @@ describe("Check successful USFM-USJ conversion for positive samples", () => {
         const testParser = initialiseParser(inputUsfmPath)
         assert(testParser instanceof USFMParser)
         const usj = testParser.toUSJ();
-        assert(testParser instanceof Object);
+        assert(usj instanceof Object);
         assert.strictEqual(usj["type"], "USJ"); 
         assert.strictEqual(usj["version"], "3.1");
         assert.strictEqual(usj.content[0].type, "book"); 
diff --git a/web-usfm-parser/test/test_list_conversion.js b/web-usfm-parser/test/test_list_conversion.js
new file mode 100644
index 00000000..02a2e317
--- /dev/null
+++ b/web-usfm-parser/test/test_list_conversion.js
@@ -0,0 +1,71 @@
+import assert from 'assert';
+import fs from 'node:fs';
+import {allUsfmFiles, initialiseParser, isValidUsfm, excludeUSJs, findAllMarkers} from './config.js';
+import {USFMParser, Filter} from '../src/index.js';
+
+
+describe("Check successful USFM-List conversion for positive samples", () => {
+
+  allUsfmFiles.forEach(function(value) {
+    if (isValidUsfm[value]) {
+      it(`Convert ${value} to List`, async (inputUsfmPath=value) => {
+        //Tests if input parses without errors
+        const testParser = await initialiseParser(inputUsfmPath)
+        assert(testParser instanceof USFMParser)
+        const list = testParser.toList();
+        assert(list instanceof Array);
+        assert.deepStrictEqual(list[0],
+        			[ 'Book', 'Chapter', 'Verse', 'Text', 'Type', 'Marker' ]); 
+
+      });
+    }
+  });
+});
+
+
+describe("Test Exclude Marker option in List conversion", () => {
+    // Test Exclude Maker option by checking markers in the List
+    const excludeTests = [
+            ['s', 'r']
+            ]
+    excludeTests.forEach(function(exList) {
+        allUsfmFiles.forEach(function(value) {
+          if (isValidUsfm[value]) {
+            it(`Exclude ${exList.slice(0, 5)} from ${value}`, async (inputUsfmPath=value) => {
+                const testParser = await initialiseParser(inputUsfmPath)
+                assert(testParser instanceof USFMParser)
+                const list = testParser.toList(exList);
+                assert(list instanceof Array);
+
+                const allTypes = list.map(row => row[5]);
+                let types = new Set(allTypes);
+                let intersection = exList.filter(value => types.has(value));
+                assert.deepStrictEqual(intersection, [])
+            });
+          }
+        })
+    })
+});
+
+describe("Test include Marker option in List conversion", () => {
+    // Test include Maker option by checking markers in the List
+    const includeTests = [
+            ['id', 'c', 'v']+Filter.TEXT+Filter.PARAGRAPHS
+            ]
+    includeTests.forEach(function(inList) {
+        allUsfmFiles.forEach(function(value) {
+          if (isValidUsfm[value]) {
+            it(`include ${inList.slice(0, 5)} of ${value} in List`, async (inputUsfmPath=value) => {
+                const testParser = await initialiseParser(inputUsfmPath)
+                assert(testParser instanceof USFMParser)
+                const list = testParser.toList(null, inList);
+                assert(list instanceof Array);
+
+                const allTypes = list.slice(1).map(row => row[5]);
+                assert( allTypes.every(element => inList.includes(element)), allTypes)
+
+            });
+          }
+        })
+    })
+});
\ No newline at end of file