Repository: sensein/cmixf

Issue:
Liters are not validated when combined with prefix
```
# 1 Liter works
cmixf > 1L
type='REAL', value='1'
type='UNITC', value='L'
1L

# 1mL does not work
cmixf > 1mL
type='REAL', value='1'
type='UNITC', value='m'
type='UNITC', value='L'
sly: Syntax error at line 1, token=UNITC
Traceback (most recent call last):
  File "/home/stefanappelhoff/miniconda3/bin/cmixf", line 11, in <module>
    load_entry_point('cmixf', 'console_scripts', 'cmixf')()
  File "/home/stefanappelhoff/Desktop/bids/cmixf/cmixf/parser.py", line 219, in main
    print(parser.parse(lexer.tokenize(text)))
  File "/home/stefanappelhoff/miniconda3/lib/python3.7/site-packages/sly/yacc.py", line 2119, in parse
    tok = self.error(errtoken)
  File "/home/stefanappelhoff/Desktop/bids/cmixf/cmixf/parser.py", line 205, in error
    raise RuntimeError(f"Could not parse {t}")
RuntimeError: Could not parse Token(type='UNITC', value='L', lineno=1, index=2)
```

Comments:

Patch:
diff --git a/README.md b/README.md
index 70f56a0..8227578 100644
--- a/README.md
+++ b/README.md
@@ -4,43 +4,50 @@
 
 A library to parse [CMIXF-12](https://people.csail.mit.edu/jaffer/MIXF/CMIXF-12).
 
-This will install a CLI `cmixf` when installed.
+## For users:
 
-At present this assumes that a number is provided before any unit.
+To install from PyPi.
 
-## For users:
+```
+$ pip install cmixf 
+```
+
+This will install a command line interface called `cmixf`.
+
+```
+$ cmixf --help
+Usage: cmixf [OPTIONS] [TEXT]...
+
+Options:
+  -d, --debug  Turn on token debugging
+  --help       Show this message and exit.
+```
+
+At present this assumes that a number is provided before any unit. To better 
+understand which characters are erroneous you can turn on the debug flag.
+
+1. Without debugging on:
 
 ```
-$ pip install cmixf # requires a python 3 environment 
 $ cmixf
-cmixf > 1mV
-type='REAL', value='1'
-type='SUBMULTIB', value='mV'
+cmixf (Ctrl+d to quit) > 1mV
 1mV
-cmixf > 1oC
-type='REAL', value='1'
-type='UNITC', value='oC'
+cmixf (Ctrl+d to quit) > 1oC
 1oC
-cmixf > <Ctrl+D to exit>
+cmixf (Ctrl+d to quit) > 1mM
+FAILED:  Line 1: Bad character 'M'
+cmixf (Ctrl+d to quit) > 
 ```
 
-If it fails it will raise an error and exit. 
+2. With debugging turned on:
 
 ```
-$ cmixf
-cmixf > 1mM
+$ cmixf --debug
+cmixf (Ctrl+d to quit) > 1mM
 type='REAL', value='1'
 type='UNITC', value='m'
-Traceback (most recent call last):
-  File "/Users/satra/software/miniconda3/envs/mixf/bin/cmixf", line 11, in <module>
-    load_entry_point('cmixf', 'console_scripts', 'cmixf')()
-  File "/Users/satra/software/sensein/cmixf/cmixf/parser.py", line 217, in main
-    for tok in lexer.tokenize(text):
-  File "/Users/satra/software/miniconda3/envs/mixf/lib/python3.8/site-packages/sly/lex.py", line 443, in tokenize
-    tok = self.error(tok)
-  File "/Users/satra/software/sensein/cmixf/cmixf/parser.py", line 54, in error
-    raise ValueError("Line %d: Bad character %r" % (self.lineno, t.value[0]))
-ValueError: Line 1: Bad character 'M'
+FAILED:  Line 1: Bad character 'M'
+cmixf (Ctrl+d to quit) > 
 ```
 
 ## For developers:
diff --git a/cmixf/parser.py b/cmixf/parser.py
index 516a614..dd929be 100644
--- a/cmixf/parser.py
+++ b/cmixf/parser.py
@@ -1,19 +1,72 @@
+import click
 from sly import Lexer, Parser
 
-decimalsubmultiple = r"(a|c|d|f|m|n|p|(u|µ)|y|z)"
-decimalmultiple = r"(E|G|M|P|T|Y|Z|da|h|k)"
-binary = r"(Ei|Gi|Ki|Mi|Pi|Ti)"
-unitc = {"unitb1": r"[A-Z][A-Z][A-Z]|A|Bq|C",
-         "units": r"dB|d|h|min|u",
-         "unitb2": r"F|Gy|Hz|H|J|K|Np|N|(Ohm|Ω)|Pa|Sv|S|T|V|W|Wb",
-         "unitn": r"L|(oC|°C)|(°|o)|rad|sr",
-         "unitb3": r"bit|cd|eV|g|kat|lm|lx|mol|m|s",
-         "unitp": r"Bd|B|r|t"}
-unitb = r"(" + r'|'.join([unitc["unitb1"], unitc["unitb2"],
-                          unitc["unitb3"]]) + r")"
-unitn = r"(" + unitc["unitn"] + r")"
-unitp = r"(" + unitc["unitp"] + r")"
-units = r"(" + unitc["units"] + r")"
+decimal_multiple_prefix = ['E', 'G', 'M', 'P', 'T', 'Y', 'Z', 'da', 'h', 'k']
+decimal_submultiple_prefix = ['a', 'c', 'd', 'f', 'm', 'n', 'p', ['u', 'µ'],
+                              'y', 'z']
+binary_prefix = ['Ei', 'Gi', 'Ki', 'Mi', 'Pi', 'Ti']
+unit_p_symbol = ['Bd', 'B', 'r', 't']
+unit_n_symbol = ['L', 'Np', ['oC', '°C'], ['o', '°'], 'rad', 'sr']
+unit_b_symbol = ['A', 'Bq', 'C', 'F', 'Gy', 'Hz', 'H', 'J', 'K', 'N',
+                 ['Ohm', 'Ω'], 'Pa', 'Sv', 'S', 'T', 'V', 'Wb', 'W', 'bit',
+                 'cd', 'eV', 'g', 'kat', 'lm', 'lx', 'mol', 'm', 's',
+                 ]
+unit_b_N_index = unit_b_symbol.index('N')
+unit___symbol = ['dB', 'd', 'h', 'min', 'u']
+currency = ['[A-Z][A-Z][A-Z]']
+
+
+def encapsulate(value_list):
+    if all([isinstance(val, str) for val in value_list]):
+        return r'(' + r'|'.join(value_list) + r')'
+    return encapsulate([val if isinstance(val, str) else encapsulate(val) for val in value_list])
+
+
+def to_regex(components):
+    if not isinstance(components, list):
+        components = [components]
+    return r''.join([encapsulate(comp) for comp in components])
+
+
+def to_list(unit_list):
+    out = []
+    for val in unit_list:
+        if isinstance(val, str):
+            out.append(val)
+        else:
+            out.extend(val)
+    return out
+
+
+def create_combos():
+    """
+    punit
+        : decimal_multiple_prefix unit_p_symbol
+        | decimal_submultiple_prefix unit_n_symbol
+        | decimal_multiple_prefix unit_b_symbol
+        | decimal_submultiple_prefix unit_b_symbol
+        | binary_prefix 'B'
+        | binary_prefix 'bit'
+        | unit_p_symbol
+        | unit_n_symbol
+        | unit_b_symbol
+        | unit___symbol
+        ;
+    """
+    def combine(list1, list2):
+        combos = []
+        for val1 in to_list(list1):
+            for val2 in to_list(list2):
+                combos.append(val1 + val2)
+        return combos
+    combos = combine(decimal_multiple_prefix, unit_p_symbol) + \
+             combine(decimal_submultiple_prefix, unit_n_symbol) + \
+             combine(decimal_multiple_prefix, unit_b_symbol) + \
+             combine(decimal_submultiple_prefix, unit_b_symbol) + \
+             combine(binary_prefix, ["B", "bit"]) + \
+             to_list(unit_p_symbol) + to_list(unit_n_symbol) + \
+             to_list(unit_b_symbol) + to_list(unit___symbol)
+    return ["1"+val for val in combos]
 
 
 class CMIXFLexer(Lexer):
@@ -42,13 +95,19 @@ class CMIXFLexer(Lexer):
     EXP = r"\^"
     REAL = r"-?\d*\.?\d+(?:[eE][-+]?\d+)?"
     DOT = r"\."
-    BIT = binary + r"bit"
-    BYTE = binary + r"B"
-    MULTIB = decimalmultiple + unitb
-    SUBMULTIB = decimalsubmultiple + unitb
-    MULTIP = decimalmultiple + unitp
-    UNITC = r'|'.join([val for val in unitc.values()])
-    SUBMULTIN = decimalsubmultiple + unitn
+    BIT = to_regex([binary_prefix, [r"bit"]])
+    BYTE = to_regex([binary_prefix, [r"B"]])
+    SUBMULTIN = r"mol|" + to_regex([decimal_submultiple_prefix, unit_n_symbol])
+    MULTIB = to_regex([decimal_multiple_prefix, currency + unit_b_symbol])
+    SUBMULTIB = to_regex([decimal_submultiple_prefix, currency + unit_b_symbol])
+    MULTIP = to_regex([decimal_multiple_prefix, unit_p_symbol])
+    UNITC = r'|'.join([encapsulate(val)[1:-1] for val in [currency + unit_b_symbol[:unit_b_N_index],
+                                                          unit___symbol,
+                                                          unit_n_symbol,
+                                                          unit_b_symbol[unit_b_N_index:],
+                                                          unit_p_symbol,
+                                                          ]
+                       ])
 
     def error(self, t):
         raise ValueError("Line %d: Bad character %r" % (self.lineno, t.value[0]))
@@ -205,15 +264,31 @@ class CMIXFParser(Parser):
         raise RuntimeError(f"Could not parse {t}")
 
 
-def main():
+def parse(text, debug):
     lexer = CMIXFLexer()
     parser = CMIXFParser()
+    tokens = lexer.tokenize(text)
+    if debug:
+        for tok in tokens:
+            print("type=%r, value=%r" % (tok.type, tok.value))
+    print(parser.parse(tokens))
+
+
+@click.command()
+@click.option("-d", "--debug", is_flag=True, help="Turn on token debugging")
+@click.argument("text", nargs=-1)
+def main(debug, text):
+    if text:
+        for val in text:
+            parse(val, debug)
+        return
     while True:
         try:
-            text = input("cmixf > ")
+            text = input("cmixf (Ctrl+d to quit) > ")
         except EOFError:
             break
         if text:
-            for tok in lexer.tokenize(text):
-                print("type=%r, value=%r" % (tok.type, tok.value))
-            print(parser.parse(lexer.tokenize(text)))
+            try:
+                parse(text, debug)
+            except Exception as e:
+                print("FAILED: ", e)


Test patch:
diff --git a/cmixf/tests/test_cmixf.py b/cmixf/tests/test_cmixf.py
index e519fa7..26b65db 100644
--- a/cmixf/tests/test_cmixf.py
+++ b/cmixf/tests/test_cmixf.py
@@ -1,6 +1,6 @@
 import pytest
 
-from ..parser import CMIXFLexer, CMIXFParser
+from ..parser import CMIXFLexer, CMIXFParser, create_combos
 
 
 @pytest.mark.parametrize(
@@ -80,6 +80,9 @@ def test_real(text, exception):
         "1J/(mol.K)",
         "1nV/Hz^(1/2)",
         "1Mibit/s",
+        "1ms",
+        "1kBq",
+        "1mL",
     ],
 )
 def test_cmixf_examples(text):
@@ -94,6 +97,8 @@ def test_cmixf_examples(text):
         "1µm",
         "1°C",
         "1Ω",
+        "1µV",
+        "1uV",
     ],
 )
 def test_bids_chars(text):
@@ -104,14 +109,10 @@ def test_bids_chars(text):
 
 @pytest.mark.parametrize(
     "text",
-    [
-        "1µV",
-        "1uV",
-        "1ms",
-        "1kBq",
-    ],
+    create_combos()
 )
-def test_new_errors(text):
+def test_combos(text):
     lexer = CMIXFLexer()
     parser = CMIXFParser()
     assert isinstance(parser.parse(lexer.tokenize(text)), str)
+