Repository: thammegowda/mtdata

Issue:
Newline and tab chars inside segments (e.g. XML formats)
Some formats may have newline and tab chars inside segments, which could result in line count mismatch. here is one. 
```
$ python -m mtdata get -l nor-slk -tr EAC_Reference -o tmp/eac-nor-slk 

$ wc -l tmp/eac-nor-slk/train-parts/EAC_Reference-nor_slk.*   
     524 tmp/eac-nor-slk/train-parts/EAC_Reference-nor_slk.nor
     531 tmp/eac-nor-slk/train-parts/EAC_Reference-nor_slk.slk
```

Discussion:

Patch:
diff --git a/mtdata/data.py b/mtdata/data.py
index ef25e02..4b5f909 100644
--- a/mtdata/data.py
+++ b/mtdata/data.py
@@ -140,6 +140,8 @@ class Dataset:
                     continue
                 if swap:
                     sent2, sent1 = sent1, sent2
+                sent1 = sent1.replace('\n', ' ').replace('\t', ' ')
+                sent2 = sent2.replace('\n', ' ').replace('\t', ' ')
                 f1.write(f'{sent1}\n')
                 f2.write(f'{sent2}\n')
                 count += 1
@@ -149,4 +151,5 @@ class Dataset:
                 log.warning(msg)
             if noise > 0:
                 log.info(f"{entry}: Noise : {noise:,}/{count:,} => {100*noise/count:.4f}%")
+            log.info(f"wrote {count} lines to {l1} == {l2}")
         return count, skips
diff --git a/mtdata/index/eu.py b/mtdata/index/eu.py
index b9f8a47..9f9a2fb 100644
--- a/mtdata/index/eu.py
+++ b/mtdata/index/eu.py
@@ -8,33 +8,28 @@ from mtdata.index import Index, Entry
 def load_all(index: Index):
     # === ECDC ===
     # https://ec.europa.eu/jrc/en/language-technologies/ecdc-translation-memory
-    cite="""@article{Steinberger2014,
-    title={An overview of the European Union's highly multilingual parallel corpora},
-    journal={Language Resources and Evaluation},
-    author={Steinberger, Ralf and Ebrahim, Mohamed and Poulis, Alexandros and Carrasco-Benitez, Manuel and Schlüter, Patrick and Przybyszewski, Marek and Gilbro, Signe},
-    year={2014},
-    doi={10.1007/s10579-014-9277-0},
-    url={https://ec.europa.eu/jrc/sites/jrcsh/files/2014_08_LRE-Journal_JRC-Linguistic-Resources_Manuscript.pdf},
-    pages={679--707},
-}"""
-    langs = ["en", "bg", "cs", "da", "de", "el", "es", "et", "fi", "fr", "ga", "hu", "is", "it", "lt", "lv", "mt", "nl", "no", "pl", "pt", "ro", "sk", "sl", "sv"]
+    cite = index.ref_db.get_bibtex('Steinberger2014')
+    langs = 'en bg cs da de el es et fi fr ga hu is it lt lv mt nl no pl pt ro sk sl sv'.split()
     for i, l1 in enumerate(langs):
         for l2 in langs[i+1:]:
-            ent = Entry(langs=(l1, l2), url="http://optima.jrc.it/Resources/ECDC-TM/ECDC-TM.zip", name="ECDC", in_ext='tmx', cite=cite, in_paths=["ECDC-TM/ECDC.tmx"])
+            ent = Entry(langs=(l1, l2), url="http://optima.jrc.it/Resources/ECDC-TM/ECDC-TM.zip",
+                        name="ECDC", in_ext='tmx', cite=cite, in_paths=["ECDC-TM/ECDC.tmx"])
             index.add_entry(ent)
 
     # === EAC ===
     # https://ec.europa.eu/jrc/en/language-technologies/eac-translation-memory
     # This corpus has two 
-    langs = ["bg", "cs", "da", "de", "el", "en", "es", "et", "fi", "fr", "hu", "is", "it", "lt", "lv", "mt", "nb", "nl", "pl", "pt", "ro", "sk", "sl", "sv", "tr"]
+    langs = 'bg cs da de el en es et fi fr hu is it lt lv mt nb nl pl pt ro sk sl sv tr'.split()
     for i, l1 in enumerate(langs):
         for l2 in langs[i+1:]:
-            ent = Entry(langs=(l1, l2), url="https://wt-public.emm4u.eu/Resources/EAC-TM/EAC-TM-all.zip", name="EAC_Forms", in_ext='tmx', cite=cite, in_paths=["EAC_FORMS.tmx"])
+            ent = Entry(langs=(l1, l2), url="https://wt-public.emm4u.eu/Resources/EAC-TM/EAC-TM-all.zip",
+                        name="EAC_Forms", in_ext='tmx', cite=cite, in_paths=["EAC_FORMS.tmx"])
             index.add_entry(ent)
-    langs = ["bg", "cs", "da", "de", "el", "en", "es", "et", "fi", "fr", "hr", "hu", "is", "it", "lt", "lv", "mt", "nl", "no", "pl", "pt", "ro", "sk", "sl", "sv", "tr"]
+    langs = 'bg cs da de el en es et fi fr hr hu is it lt lv mt nl no pl pt ro sk sl sv tr'.split()
     for i, l1 in enumerate(langs):
         for l2 in langs[i+1:]:
-            ent = Entry(langs=(l1, l2), url="https://wt-public.emm4u.eu/Resources/EAC-TM/EAC-TM-all.zip", name="EAC_Reference", in_ext='tmx', cite=cite, in_paths=["EAC_REFRENCE_DATA.tmx"])
+            ent = Entry(langs=(l1, l2), url="https://wt-public.emm4u.eu/Resources/EAC-TM/EAC-TM-all.zip",
+                        name="EAC_Reference", in_ext='tmx', cite=cite, in_paths=["EAC_REFRENCE_DATA.tmx"])
             index.add_entry(ent)
 
     # Note: DGT-TM is already in via OPUS.
diff --git a/mtdata/index/refs.bib b/mtdata/index/refs.bib
index eeaec24..dbc57b5 100644
--- a/mtdata/index/refs.bib
+++ b/mtdata/index/refs.bib
@@ -352,4 +352,14 @@
     eprint = {1907.05791},
     timestamp = {Wed, 17 Jul 2019 10:27:36 +0200},
     biburl = {https://dblp.org/rec/journals/corr/abs-1907-05791.bib},
-    bibsource = {dblp computer science bibliography, https://dblp.org} }
\ No newline at end of file
+    bibsource = {dblp computer science bibliography, https://dblp.org} }
+
+@article{Steinberger2014,
+    title={An overview of the European Union's highly multilingual parallel corpora},
+    journal={Language Resources and Evaluation},
+    author={Steinberger, Ralf and Ebrahim, Mohamed and Poulis, Alexandros and Carrasco-Benitez, Manuel and Schlüter, Patrick and Przybyszewski, Marek and Gilbro, Signe},
+    year={2014},
+    doi={10.1007/s10579-014-9277-0},
+    url={https://ec.europa.eu/jrc/sites/jrcsh/files/2014_08_LRE-Journal_JRC-Linguistic-Resources_Manuscript.pdf},
+    pages={679--707},
+}
\ No newline at end of file
diff --git a/mtdata/tmx.py b/mtdata/tmx.py
index 0e81fd8..a287c8f 100644
--- a/mtdata/tmx.py
+++ b/mtdata/tmx.py
@@ -40,7 +40,7 @@ def parse_tmx(data, log_every=DEF_PROGRESS):
             t = time.time()
         tu.clear()
 
-def read_tmx(path: Union[Path], langs=None):
+def read_tmx(path: Union[Path, str], langs=None):
     """
     reads a TMX file as records
     :param path: path to .tmx file
@@ -71,8 +71,8 @@ def read_tmx(path: Union[Path], langs=None):
         log.warning(f"Skipped {fails} entries due to language mismatch in TMX {path}")
     log.info(f"Extracted {passes} pairs from TMX {path}")
 
-def main(inp, out):
-    recs = read_tmx(inp)
+def main(inp, out, langs):
+    recs = read_tmx(inp, langs=langs)
     with IO.writer(out) as out:
         count = 0
         for rec in recs:
@@ -81,6 +81,8 @@ def main(inp, out):
             count += 1
         log.warning(f"Wrote {count} lines to {out}")
 
+def split_tuple(text: str):
+    return tuple(text.split("-"))
 
 if __name__ == '__main__':
     p = argparse.ArgumentParser(description='A tool to convert TMX to TSV',
@@ -88,5 +90,8 @@ if __name__ == '__main__':
     p.add_argument('-i', '--inp', type=Path, required=True, help='Input file path')
     p.add_argument('-o', '--out', type=Path, default=Path('/dev/stdout'),
                    help='Output file path')
+    p.add_argument('-l', '--langs', type=split_tuple, default=None,
+                   help='Languages from TMX. example: eng-fra or en-fr')
+
     args = vars(p.parse_args())
     main(**args)


Test patch:
diff --git a/tests/test_tmx.py b/tests/test_tmx.py
index 4ce8617..dead81d 100644
--- a/tests/test_tmx.py
+++ b/tests/test_tmx.py
@@ -27,4 +27,16 @@ def test_rapid1019():
         assert ces[0].strip() =='(^) Výrobky, které byly určeny k prodeji před 20. květnem 2016, mohou být v EU stále prodávány do roku 2017.'
 
 
-test_rapid1019()
\ No newline at end of file
+def test_eac_ref():
+    with TemporaryDirectory() as out_dir:
+        out_dir = Path(out_dir)
+        args = Namespace(cache=mtdata.cache_dir, langs=('nor', 'slk'), merge=False,
+                         out=out_dir, task='get', test_names=None, train_names=['EAC_Reference'])
+        get_data(args)
+        parts_dir = out_dir / 'train-parts'
+        nor = parts_dir / 'EAC_Reference-nor_slk.nor'
+        slk = parts_dir / 'EAC_Reference-nor_slk.slk'
+        nor = nor.read_text().splitlines()
+        slk = slk.read_text().splitlines()
+        print('nor, slk == ', len(nor), len(slk))
+        assert len(nor) == len(slk)