Repository: sensein/cmixf

Issue:
fix corner cases that depend on ordering of tokens
the use of sly makes tokenizing dependent on order of tokens. this results in a set of units to be not parsed correctly.

https://github.com/sensein/cmixf/blob/master/cmixf/tests/test_cmixf.py#L84


Comments:

Patch:
diff --git a/README.md b/README.md
index 3788677..5958bbb 100644
--- a/README.md
+++ b/README.md
@@ -4,6 +4,10 @@
 
 A library to parse [CMIXF-12](https://people.csail.mit.edu/jaffer/MIXF/CMIXF-12).
 
+This will install a CLI `cmixf` when installed.
+
+At present this assumes that a number is provided before any unit.
+
 ## For developers:
 
 After cloning the repo do:
diff --git a/cmixf/parser.py b/cmixf/parser.py
index 209e82f..db429d4 100644
--- a/cmixf/parser.py
+++ b/cmixf/parser.py
@@ -2,13 +2,17 @@ from sly import Lexer, Parser
 
 decimalsubmultiple = r"(a|c|d|f|m|n|p|u|y|z)"
 decimalmultiple = r"(E|G|M|P|T|Y|Z|da|h|k)"
-unitb = (
-    r"([A-Z][A-Z][A-Z]|A|Bq|C|F|Gy|H|Hz|J|K|N|Ohm|Pa|Sv|S|T|V|W|Wb"
-    r"|bit|cd|eV|g|kat|lm|lx|mol|m|s)"
-)
-unitn = r"(L|Np|oC|o|rad|sr)"
-unitp = r"(Bd|B|r|t)"
 binary = r"(Ei|Gi|Ki|Mi|Pi|Ti)"
+unitc = [r"[A-Z][A-Z][A-Z]|A|Bq|C",
+         r"dB|d|h|min|u",
+         r"F|Gy|Hz|H|J|K|Np|N|(Ohm|Ω)|Pa|Sv|S|T|V|W|Wb",
+         r"L|(oC|°C)|o|rad|sr",
+         r"bit|cd|eV|g|kat|lm|lx|mol|m|s|µ",
+         r"Bd|B|r|t"]
+unitb = r"(" + r'|'.join([unitc[0], unitc[2], unitc[4]]) + r")"
+unitn = r"(" + unitc[3] + r")"
+unitp = r"(" + unitc[5] + r")"
+units = r"(" + unitc[2] + r")"
 
 
 class CMIXFLexer(Lexer):
@@ -19,10 +23,7 @@ class CMIXFLexer(Lexer):
         SUBMULTIN,
         MULTIB,
         SUBMULTIB,
-        UNITP,
-        UNITN,
-        UNITB,
-        UNITS,
+        UNITC,
         DOT,
         DIV,
         LPAREN,
@@ -40,16 +41,13 @@ class CMIXFLexer(Lexer):
     EXP = r"\^"
     REAL = r"-?\d*\.?\d+(?:[eE][-+]?\d+)?"
     DOT = r"\."
+    BIT = binary + r"bit"
+    BYTE = binary + r"B"
+    UNITC = r'|'.join(unitc)
     MULTIP = decimalmultiple + unitp
     SUBMULTIN = decimalsubmultiple + unitn
     MULTIB = decimalmultiple + unitb
     SUBMULTIB = decimalsubmultiple + unitb
-    BIT = binary + r"bit"
-    BYTE = binary + r"B"
-    UNITS = r"(dB|d|h|min|u)"
-    UNITB = unitb
-    UNITN = unitn
-    UNITP = unitp
 
     def error(self, t):
         raise ValueError("Line %d: Bad character %r" % (self.lineno, t.value[0]))
@@ -193,21 +191,9 @@ class CMIXFParser(Parser):
     def punit(self, p):
         return p.BIT
 
-    @_("UNITP")
-    def punit(self, p):
-        return p.UNITP
-
-    @_("UNITN")
-    def punit(self, p):
-        return p.UNITN
-
-    @_("UNITB")
-    def punit(self, p):
-        return p.UNITB
-
-    @_("UNITS")
+    @_("UNITC")
     def punit(self, p):
-        return p.UNITS
+        return p.UNITC
 
     @_("REAL")
     def real(self, p):


Test patch:
diff --git a/cmixf/tests/test_cmixf.py b/cmixf/tests/test_cmixf.py
index 11f51f7..4541261 100644
--- a/cmixf/tests/test_cmixf.py
+++ b/cmixf/tests/test_cmixf.py
@@ -73,6 +73,13 @@ def test_real(text, exception):
         "1EUR/kg",
         "1JPY/USD",
         "1",
+        "1mol/m^3",
+        "1W/sr",
+        "1W/(m^2.sr)",
+        "1J/mol",
+        "1J/(mol.K)",
+        "1nV/Hz^(1/2)",
+        "1Mibit/s",
     ],
 )
 def test_cmixf_examples(text):
@@ -81,20 +88,15 @@ def test_cmixf_examples(text):
     assert isinstance(parser.parse(lexer.tokenize(text)), str)
 
 
-@pytest.mark.xfail()
 @pytest.mark.parametrize(
     "text",
     [
-        "1mol/m^3",
-        "1W/sr",
-        "1W/(m^2.sr)",
-        "1J/mol",
-        "1J/(mol.K)",
-        "1nV/Hz^(1/2)",
-        "1Mib/s",
+        "1µ",
+        "1°C",
+        "1Ω",
     ],
 )
-def test_cmixf_failures(text):
+def test_bids_chars(text):
     lexer = CMIXFLexer()
     parser = CMIXFParser()
     assert isinstance(parser.parse(lexer.tokenize(text)), str)