Repository: timmahrt/praatIO

Issue:
Issues parsing TextGrids from ELAN
I've had a couple of users reporting issues with loading TextGrids exported from ELAN.  The issue seems to be that the "item [1]" lines are formatted without a space ("item[1]"), so the parsing in https://github.com/timmahrt/praatIO/blob/master/praatio/tgio.py#L1896 fails. I think a reasonable fix would be something like `re.split(r'item ?\[', data, flags=re.MULTILINE)[1:]`.

Looks like you're working on a 5.0, so don't know if that would be the place to fix it or if it would be better for me to submit a PR for the main branch.

Discussion:

Patch:
diff --git a/examples/add_tiers.py b/examples/add_tiers.py
index ef90f73..9b06542 100644
--- a/examples/add_tiers.py
+++ b/examples/add_tiers.py
@@ -13,6 +13,7 @@ if not os.path.exists(outputPath):
     os.mkdir(outputPath)
 
 tgPhones = tgio.openTextgrid(join(path, "bobby_phones.TextGrid"))
+elanTgPhones = tgio.openTextgrid(join(path, "bobby_phones_elan.TextGrid"))
 tgWords = tgio.openTextgrid(join(path, "bobby_words.TextGrid"))
 
 tgPhones.addTier(tgWords.tierDict["word"])
diff --git a/examples/files/bobby_phones_elan.TextGrid b/examples/files/bobby_phones_elan.TextGrid
new file mode 100644
index 0000000..63843ab
--- /dev/null
+++ b/examples/files/bobby_phones_elan.TextGrid
@@ -0,0 +1,74 @@
+File type = "ooTextFile"
+Object class = "TextGrid"
+
+xmin = 0.0
+xmax = 1.194625
+tiers? <exists> 
+size = 1 
+item []: 
+    item[1]:
+        class = "IntervalTier" 
+        name = "phone" 
+        xmin = 0.0
+        xmax = 1.18979591837 
+        intervals: size = 15 
+        intervals [1]
+            xmin = 0.0124716553288 
+            xmax = 0.06469123242311078 
+            text = "" 
+        intervals [2]
+            xmin = 0.06469123242311078 
+            xmax = 0.08438971390281873 
+            text = "B" 
+        intervals [3]
+            xmin = 0.08438971390281873 
+            xmax = 0.23285789838876556 
+            text = "AA1" 
+        intervals [4]
+            xmin = 0.23285789838876556 
+            xmax = 0.2788210218414174 
+            text = "B" 
+        intervals [5]
+            xmin = 0.2788210218414174 
+            xmax = 0.41156462585 
+            text = "IY0" 
+        intervals [6]
+            xmin = 0.41156462585 
+            xmax = 0.47094510353588265 
+            text = "R" 
+        intervals [7]
+            xmin = 0.47094510353588265 
+            xmax = 0.521315192744 
+            text = "IH1" 
+        intervals [8]
+            xmin = 0.521315192744 
+            xmax = 0.658052967538796 
+            text = "PT" 
+        intervals [9]
+            xmin = 0.658052967538796 
+            xmax = 0.680952380952 
+            text = "DH" 
+        intervals [10]
+            xmin = 0.680952380952 
+            xmax = 0.740816326531 
+            text = "AH0" 
+        intervals [11]
+            xmin = 0.740816326531 
+            xmax = 0.807647261005538 
+            text = "L" 
+        intervals [12]
+            xmin = 0.807647261005538 
+            xmax = 0.910430839002 
+            text = "EH1" 
+        intervals [13]
+            xmin = 0.910430839002 
+            xmax = 0.980272108844 
+            text = "JH" 
+        intervals [14]
+            xmin = 0.980272108844 
+            xmax = 1.1171482864527198 
+            text = "ER0" 
+        intervals [15]
+            xmin = 1.1171482864527198 
+            xmax = 1.18979591837 
+            text = "" 
diff --git a/examples/files/bobby_words_with_newlines_longfile_elan.TextGrid b/examples/files/bobby_words_with_newlines_longfile_elan.TextGrid
new file mode 100644
index 0000000..f1fcdfa
--- /dev/null
+++ b/examples/files/bobby_words_with_newlines_longfile_elan.TextGrid
@@ -0,0 +1,82 @@
+File type = "ooTextFile"
+Object class = "TextGrid"
+
+xmin = 0 
+xmax = 1.194625 
+tiers? <exists> 
+size = 3 
+item []: 
+    item[1]:
+        class = "IntervalTier" 
+        name = """word""" 
+        xmin = 0 
+        xmax = 1.194625 
+        intervals: size = 6 
+        intervals [1]
+            xmin = 0 
+            xmax = 0.06469123242311078 
+            text = "" 
+        intervals [2]
+            xmin = 0.06469123242311078 
+            xmax = 0.41156462585 
+            text = """""""BOBBY""""""
+Noun" 
+        intervals [3]
+            xmin = 0.41156462585 
+            xmax = 0.6576881808447274 
+            text = "RIPPED
+Verb" 
+        intervals [4]
+            xmin = 0.6576881808447274 
+            xmax = 0.740816326531 
+            text = "THE
+Determiner" 
+        intervals [5]
+            xmin = 0.740816326531 
+            xmax = 1.1171482864527198 
+            text = "LEDGER
+Noun" 
+        intervals [6]
+            xmin = 1.1171482864527198 
+            xmax = 1.194625 
+            text = "" 
+    item[2]:
+        class = "IntervalTier" 
+        name = "phrase" 
+        xmin = 0 
+        xmax = 1.194625 
+        intervals: size = 3 
+        intervals [1]
+            xmin = 0 
+            xmax = 0.06469123242311078 
+            text = "" 
+        intervals [2]
+            xmin = 0.06469123242311078 
+            xmax = 1.1171482864527198 
+            text = "BOBBY RIPPED THE LEDGER" 
+        intervals [3]
+            xmin = 1.1171482864527198 
+            xmax = 1.194625 
+            text = "" 
+    item[3]:
+        class = "TextTier" 
+        name = "" 
+        xmin = 0 
+        xmax = 1.194625 
+        points: size = 4 
+        points [1]
+            number = 0.23290458517889742 
+            mark = "133
+p1" 
+        points [2]
+            number = 0.5304541883551366 
+            mark = "0
+p2" 
+        points [3]
+            number = 0.6966964767693916 
+            mark = "93
+p3" 
+        points [4]
+            number = 0.9231714783772174 
+            mark = "85
+p4" 
diff --git a/praatio/tgio.py b/praatio/tgio.py
index e5f0afc..aa64649 100644
--- a/praatio/tgio.py
+++ b/praatio/tgio.py
@@ -1861,7 +1861,7 @@ def openTextgrid(fnFullPath, readRaw=False, readAsJson=False):
         data = data.replace("\r\n", "\n")
 
         caseA = "ooTextFile short" in data
-        caseB = "item [" not in data
+        caseB = not re.search(r"item ?\[", data)
         if caseA or caseB:
             textgrid = _parseShortTextgrid(data)
         else:
@@ -1883,7 +1883,7 @@ def _parseNormalTextgrid(data):
     newTG = Textgrid()
 
     # Toss textgrid header
-    header, data = data.split("item [", 1)
+    header, data = re.split(r'item ?\[', data, maxsplit=1, flags=re.MULTILINE)
 
     headerList = header.split("\n")
     tgMin = float(headerList[3].split("=")[1].strip())
@@ -1893,59 +1893,58 @@ def _parseNormalTextgrid(data):
     newTG.maxTimestamp = tgMax
 
     # Process each tier individually (will be output to separate folders)
-    tierList = data.split("item [")[1:]
+    tierList = re.split(r"item ?\[", data, flags=re.MULTILINE)[1:]
     for tierTxt in tierList:
 
         hasData = True
 
         if 'class = "IntervalTier"' in tierTxt:
             tierType = INTERVAL_TIER
-            searchWord = "intervals ["
+            searchWord = r"intervals ?\["
         else:
             tierType = POINT_TIER
-            searchWord = "points ["
+            searchWord = r"points ?\["
 
         # Get tier meta-information
         try:
-            header, tierData = tierTxt.split(searchWord, 1)
+            d = re.split(searchWord, tierTxt, flags=re.MULTILINE)
+            header, tierData = d[0], d[1:]
         except ValueError:
             # A tier with no entries
-            if "size = 0" in tierTxt:
+            if re.search(r"size ?= ?0", tierTxt):
                 header = tierTxt
                 tierData = ""
                 hadData = False
             else:
                 raise
-        tierName = header.split("name = ")[1].split("\n", 1)[0]
-        tierName, tierNameI = _fetchTextRow(header, 0, "name = ")
-        tierStart = header.split("xmin = ")[1].split("\n", 1)[0]
+        tierName = re.search(r"name ?= ?\"(.*)\"\s*$", header, flags=re.MULTILINE).groups()[0]
+        tierName = re.sub(r'""', '"', tierName)
+
+        tierStart = re.search(r"xmin ?= ?([\d.]+)\s*$", header, flags=re.MULTILINE).groups()[0]
         tierStart = strToIntOrFloat(tierStart)
-        tierEnd = header.split("xmax = ")[1].split("\n", 1)[0]
+
+        tierEnd = re.search(r"xmax ?= ?([\d.]+)\s*$", header, flags=re.MULTILINE).groups()[0]
         tierEnd = strToIntOrFloat(tierEnd)
 
         # Get the tier entry list
         tierEntryList = []
         labelI = 0
         if tierType == INTERVAL_TIER:
-            while True:
-                try:
-                    timeStart, timeStartI = _fetchRow(tierData, labelI, "xmin = ")
-                    timeEnd, timeEndI = _fetchRow(tierData, timeStartI, "xmax = ")
-                    label, labelI = _fetchTextRow(tierData, timeEndI, "text = ")
-                except (ValueError, IndexError):
-                    break
+            for element in tierData:
+                timeStart = re.search(r"xmin ?= ?([\d.]+)\s*$", element, flags=re.MULTILINE).groups()[0]
+                timeEnd = re.search(r"xmax ?= ?([\d.]+)\s*$", element, flags=re.MULTILINE).groups()[0]
+                label = re.search(r"text ?= ?\"(.*)\"\s*$", element, flags=re.MULTILINE|re.DOTALL).groups()[0]
 
                 label = label.strip()
+                label = re.sub(r'""', '"', label)
                 tierEntryList.append((timeStart, timeEnd, label))
             tier = IntervalTier(tierName, tierEntryList, tierStart, tierEnd)
-        else:
-            while True:
-                try:
-                    time, timeI = _fetchRow(tierData, labelI, "number = ")
-                    label, labelI = _fetchTextRow(tierData, timeI, "mark = ")
-                except (ValueError, IndexError):
-                    break
 
+
+        else:
+            for element in tierData:
+                time = re.search(r"number ?= ?([\d.]+)\s*$", element, flags=re.MULTILINE).groups()[0]
+                label = re.search(r"mark ?= ?\"(.*)\"\s*$", element, flags=re.MULTILINE|re.DOTALL).groups()[0]
                 label = label.strip()
                 tierEntryList.append((time, label))
             tier = PointTier(tierName, tierEntryList, tierStart, tierEnd)


Test patch:
diff --git a/examples/test/io_tests.py b/examples/test/io_tests.py
index d309c1b..60844bb 100644
--- a/examples/test/io_tests.py
+++ b/examples/test/io_tests.py
@@ -110,6 +110,15 @@ class IOTests(unittest.TestCase):
 
         self.assertTrue(areTheSame(inputFN, outputFN, readFile))
 
+        fn = "bobby_words_with_newlines_longfile_elan.TextGrid"
+        elanInputFN = join(self.dataRoot, fn)
+        elanOutputFN = join(self.outputRoot, fn)
+
+        tg = tgio.openTextgrid(elanInputFN)
+        tg.save(elanOutputFN, useShortForm=False)
+
+        self.assertTrue(areTheSame(inputFN, elanOutputFN, readFile))
+
     def test_tg_io(self):
         """Tests for reading/writing textgrid io"""
         fn = "textgrid_to_merge.TextGrid"