Repository: Pyomo/pyomo

Issue:
dat file parser will convert strings without letters (but other characters) to integers
[bad_set.zip](https://github.com/Pyomo/pyomo/files/3644219/bad_set.zip)

The attached file contains a dat file and pyomo AbstractModel. Contents of bad_set.dat:
```
set BadSet :=
"12345_67890"
"A_12345_67890"
;
```

However, when I print the contents of BadSet to the screen (or another file) I get:
```
1234567890
A_12345_67890
```
despite marking this specifically as a string.

As a consequence, I cannot even pprint BadSet:
```
1 Set Declarations
    BadSet : Dim=0, Dimen=1, Size=2, Domain=None, Ordered=False, Bounds=None
Traceback (most recent call last):
  File "bad_set.py", line 9, in <module>
    i.pprint()
...
TypeError: '<' not supported between instances of 'str' and 'int'
```


Discussion:

Resolving patch:
diff --git a/pyomo/dataportal/parse_datacmds.py b/pyomo/dataportal/parse_datacmds.py
--- a/pyomo/dataportal/parse_datacmds.py
+++ b/pyomo/dataportal/parse_datacmds.py
@@ -2,14 +2,15 @@
 #
 #  Pyomo: Python Optimization Modeling Objects
 #  Copyright 2017 National Technology and Engineering Solutions of Sandia, LLC
-#  Under the terms of Contract DE-NA0003525 with National Technology and 
-#  Engineering Solutions of Sandia, LLC, the U.S. Government retains certain 
+#  Under the terms of Contract DE-NA0003525 with National Technology and
+#  Engineering Solutions of Sandia, LLC, the U.S. Government retains certain
 #  rights in this software.
 #  This software is distributed under the 3-clause BSD License.
 #  ___________________________________________________________________________
 
 __all__ = ['parse_data_commands']
 
+import bisect
 import sys
 import os
 import os.path
@@ -18,9 +19,12 @@
 from inspect import getfile, currentframe
 from six.moves import xrange
 
-from pyutilib.misc import flatten_list
-from pyutilib.ply import t_newline, t_ignore, _find_column, p_error, ply_init
-        
+from pyutilib.misc import flatten_list, import_file
+
+from pyomo.common import config
+from pyomo.common.fileutils import this_file_dir
+
+_re_number = r'[-+]?(?:[0-9]+\.?[0-9]*|\.[0-9]+)(?:[eE][-+]?[0-9]+)?'
 
 ## -----------------------------------------------------------
 ##
@@ -29,7 +33,6 @@
 ## -----------------------------------------------------------
 
 _parse_info = None
-debugging = False
 
 states = (
   ('data','inclusive'),
@@ -67,11 +70,13 @@
     "EQ",
     "TR",
     "ASTERISK",
+    "NUM_VAL",
     #"NONWORD",
-    "INT_VAL",
-    "FLOAT_VAL",
 ] + list(reserved.values())
 
+# Ignore space and tab
+t_ignore = " \t\r"
+
 # Regular expression rules
 t_COMMA     = r","
 t_LBRACKET  = r"\["
@@ -85,9 +90,29 @@
 t_RPAREN    = r"\)"
 t_ASTERISK  = r"\*"
 
+#
+# Notes on PLY tokenization
+#   - token functions (beginning with "t_") are prioritized in the order
+#     that they are declared in this module
+#
+def t_newline(t):
+    r'[\n]+'
+    t.lexer.lineno += len(t.value)
+    t.lexer.linepos.extend(t.lexpos+i for i,_ in enumerate(t.value))
+
 # Discard comments
+_re_singleline_comment = r'(?:\#[^\n]*)'
+_re_multiline_comment = r'(?:/\*(?:[\n]|.)*?\*/)'
+@lex.TOKEN('|'.join([_re_singleline_comment, _re_multiline_comment]))
 def t_COMMENT(t):
-    r'(\#[^\n]*)|(/\*(.*?).?(\*/))'
+    # Single-line and multi-line strings
+    nlines = t.value.count('\n')
+    t.lexer.lineno += nlines
+    # We will never need to determine column numbers within this comment
+    # block, so it is sufficient to just worry about the *last* newline
+    # in the comment
+    lastpos = t.lexpos + t.value.rfind('\n')
+    t.lexer.linepos.extend(lastpos for i in range(nlines))
 
 def t_COLONEQ(t):
     r':='
@@ -99,38 +124,32 @@ def t_SEMICOLON(t):
     t.lexer.begin('INITIAL')
     return t
 
+# Numbers must be followed by a delimiter token (EOF is not a concern,
+# as valid DAT files always end with a ';').
+@lex.TOKEN(_re_number + r'(?=[\s()\[\]{}:;,])')
+def t_NUM_VAL(t):
+    _num = float(t.value)
+    if '.' in t.value:
+        t.value = _num
+    else:
+        _int = int(_num)
+        t.value = _int if _num == _int else _num
+    return t
+
 def t_WORDWITHLBRACKET(t):
-    r'[a-zA-Z0-9_][a-zA-Z0-9_\.\-]*\['
-    if t.value in reserved:
-        t.type = reserved[t.value]    # Check for reserved words
+    r'[a-zA-Z_][a-zA-Z0-9_\.\-]*\['
     return t
 
 def t_WORD(t):
-    r'[a-zA-Z_0-9][a-zA-Z_0-9\.+\-]*'
+    r'[a-zA-Z_][a-zA-Z_0-9\.+\-]*'
     if t.value in reserved:
         t.type = reserved[t.value]    # Check for reserved words
     return t
 
 def t_STRING(t):
     r'[a-zA-Z0-9_\.+\-\\\/]+'
-    if t.value in reserved:
-        t.type = reserved[t.value]    # Check for reserved words
-    return t
-
-def t_FLOAT_VAL(t):
-    '[-+]?[0-9]+(\.([0-9]+)?([eE][-+]?[0-9]+)?|[eE][-+]?[0-9]+)'
-    try:
-        t.value = float(t.value)
-        #t.type = "FLOAT_VAL"
-        return t
-    except:
-        print("ERROR: "+t.value)
-        raise IOError
-
-def t_INT_VAL(t):
-    '[-+]?[0-9]+([eE][-+]?[0-9]+)?'
-    #t.type = "INT_VAL"
-    t.value = int(t.value)
+    # Note: RE guarantees the string has no embedded quotation characters
+    t.value = '"'+t.value+'"'
     return t
 
 def t_data_BRACKETEDSTRING(t):
@@ -138,23 +157,40 @@ def t_data_BRACKETEDSTRING(t):
     # NO SPACES
     # a[1,_df,'foo bar']
     # [1,*,'foo bar']
-    if t.value in reserved:
-        t.type = reserved[t.value]    # Check for reserved words
     return t
 
+_re_quoted_str = r'"(?:[^"]|"")*"'
+@lex.TOKEN("|".join([_re_quoted_str, _re_quoted_str.replace('"',"'")]))
 def t_QUOTEDSTRING(t):
-    r'"([^"]|\"\")*"|\'([^\']|\'\')*\''
-    if t.value in reserved:
-        t.type = reserved[t.value]    # Check for reserved words
+    # Normalize the quotes to use '"', and replace doubled ("escaped")
+    # quotation characters with a single character
+    t.value = '"' + t.value[1:-1].replace(2*t.value[0], t.value[0]) + '"'
     return t
 
 #t_NONWORD   = r"[^\.A-Za-z0-9,;:=<>\*\(\)\#{}\[\] \n\t\r]+"
 
 # Error handling rule
-def t_error(t):             #pragma:nocover
-    raise IOError("ERROR: Token %s Value %s Line %s Column %s" % (t.type, t.value, t.lineno, t.lexpos))
-    t.lexer.skip(1)
-
+def t_error(t):
+    raise IOError("ERROR: Token %s Value %s Line %s Column %s"
+                  % (t.type, t.value, t.lineno, t.lexpos))
+
+## DEBUGGING: uncomment to get tokenization information
+# def _wrap(_name, _fcn):
+#     def _wrapper(t):
+#         print(_name + ": %s" % (t.value,))
+#         return _fcn(t)
+#     _wrapper.__doc__ = _fcn.__doc__
+#     return _wrapper
+# import inspect
+# for _name in list(globals()):
+#     if _name.startswith('t_') and inspect.isfunction(globals()[_name]):
+#         globals()[_name] = _wrap(_name, globals()[_name])
+
+def _lex_token_position(t):
+    i = bisect.bisect_left(t.lexer.linepos, t.lexpos)
+    if i:
+        return t.lexpos - t.lexer.linepos[i-1]
+    return t.lexpos
 
 ## -----------------------------------------------------------
 ##
@@ -234,7 +270,7 @@ def p_statement(p):
         p[0] = [p[1]]+ [p[2]] + [p[4]]
     else:
         # Not necessary, but nice to document how statement could end up None
-        p[0] = None 
+        p[0] = None
     #print(p[0])
 
 def p_datastar(p):
@@ -249,19 +285,19 @@ def p_datastar(p):
 
 def p_data(p):
     '''
-    data : data WORD
+    data : data NUM_VAL
+         | data WORD
          | data STRING
          | data QUOTEDSTRING
          | data BRACKETEDSTRING
          | data SET
          | data TABLE
          | data PARAM
-         | data INT_VAL
-         | data FLOAT_VAL
          | data LPAREN
          | data RPAREN
          | data COMMA
          | data ASTERISK
+         | NUM_VAL
          | WORD
          | STRING
          | QUOTEDSTRING
@@ -269,8 +305,6 @@ def p_data(p):
          | SET
          | TABLE
          | PARAM
-         | INT_VAL
-         | FLOAT_VAL
          | LPAREN
          | RPAREN
          | COMMA
@@ -282,8 +316,8 @@ def p_data(p):
         tmp = p[1]
     else:
         tmp = p[2]
-    if type(tmp) is str and tmp[0] == '"' and tmp[-1] == '"' and len(tmp) > 2 and not ' ' in tmp:
-        tmp = tmp[1:-1]
+    #if type(tmp) is str and tmp[0] == '"' and tmp[-1] == '"' and len(tmp) > 2 and not ' ' in tmp:
+    #    tmp = tmp[1:-1]
 
     # Grow items list according to parsed item length
     if single_item:
@@ -307,22 +341,20 @@ def p_args(p):
 
 def p_arg(p):
     '''
-    arg : arg COMMA WORD
+    arg : arg COMMA NUM_VAL
+         | arg COMMA WORD
          | arg COMMA STRING
          | arg COMMA QUOTEDSTRING
          | arg COMMA SET
          | arg COMMA TABLE
          | arg COMMA PARAM
-         | arg COMMA INT_VAL
-         | arg COMMA FLOAT_VAL
+         | NUM_VAL
          | WORD
          | STRING
          | QUOTEDSTRING
          | SET
          | TABLE
          | PARAM
-         | INT_VAL
-         | FLOAT_VAL
     '''
     # Locate and handle item as necessary
     single_item = len(p) == 2
@@ -355,7 +387,8 @@ def p_itemstar(p):
 
 def p_items(p):
     '''
-    items : items WORD
+    items : items NUM_VAL
+          | items WORD
           | items STRING
           | items QUOTEDSTRING
           | items COMMA
@@ -372,8 +405,7 @@ def p_items(p):
           | items SET
           | items TABLE
           | items PARAM
-          | items INT_VAL
-          | items FLOAT_VAL
+          | NUM_VAL
           | WORD
           | STRING
           | QUOTEDSTRING
@@ -391,8 +423,6 @@ def p_items(p):
           | SET
           | TABLE
           | PARAM
-          | INT_VAL
-          | FLOAT_VAL
     '''
     # Locate and handle item as necessary
     single_item = len(p) == 2
@@ -413,6 +443,13 @@ def p_items(p):
         tmp_lst.append(tmp)
         p[0] = tmp_lst
 
+def p_error(p):
+    if p is None:
+        tmp = "Syntax error at end of file."
+    else:
+        tmp = "Syntax error at token '%s' with value '%s' (line %s, column %s)"\
+              % (p.type, p.value, p.lineno, _lex_token_position(p))
+    raise IOError(tmp)
 
 # --------------------------------------------------------------
 # the DAT file lexer and yaccer only need to be
@@ -430,7 +467,6 @@ def p_items(p):
 #
 def parse_data_commands(data=None, filename=None, debug=0, outputdir=None):
 
-    global debugging
     global dat_lexer
     global dat_yaccer
 
@@ -459,21 +495,21 @@ def parse_data_commands(data=None, filename=None, debug=0, outputdir=None):
                 os.remove(tabmodule+".py")
             if os.path.exists(tabmodule+".pyc"):
                 os.remove(tabmodule+".pyc")
-            debugging=True
 
         dat_lexer = lex.lex()
         #
         tmpsyspath = sys.path
         sys.path.append(outputdir)
-        dat_yaccer = yacc.yacc(debug=debug, 
-                                    tabmodule=tabmodule, 
-                                    outputdir=outputdir,
-                                    optimize=True)
+        dat_yaccer = yacc.yacc(debug=debug,
+                               tabmodule=tabmodule,
+                               outputdir=outputdir,
+                               optimize=True)
         sys.path = tmpsyspath
 
     #
     # Initialize parse object
     #
+    dat_lexer.linepos = []
     global _parse_info
     _parse_info = {}
     _parse_info[None] = []
@@ -481,32 +517,17 @@ def parse_data_commands(data=None, filename=None, debug=0, outputdir=None):
     #
     # Parse the file
     #
-    global _parsedata
-    if not data is None:
-        _parsedata=data
-        ply_init(_parsedata)
-        dat_yaccer.parse(data, lexer=dat_lexer, debug=debug)
-    elif not filename is None:
-        f = open(filename, 'r')
-        try:
-            data = f.read()
-        except Exception:
-            e = sys.exc_info()[1]
-            f.close()
-            del f
-            raise e
-        f.close()
-        del f
-        _parsedata=data
-        ply_init(_parsedata)
-        dat_yaccer.parse(data, lexer=dat_lexer, debug=debug)
-    else:
-        _parse_info = None
-    #
-    # Disable parsing I/O
-    #
-    debugging=False
-    #print(_parse_info)
+    if filename is not None:
+        if data is not None:
+            raise ValueError("parse_data_commands: cannot specify both "
+                             "data and filename arguments")
+        with open(filename, 'r') as FILE:
+            data = FILE.read()
+
+    if data is None:
+        return None
+
+    dat_yaccer.parse(data, lexer=dat_lexer, debug=debug)
     return _parse_info
 
 if __name__ == '__main__':
diff --git a/pyomo/dataportal/process_data.py b/pyomo/dataportal/process_data.py
--- a/pyomo/dataportal/process_data.py
+++ b/pyomo/dataportal/process_data.py
@@ -18,7 +18,9 @@
 import pyutilib.common
 from pyutilib.misc import flatten
 
-from pyomo.dataportal.parse_datacmds import parse_data_commands
+from pyomo.dataportal.parse_datacmds import (
+    parse_data_commands, _re_number
+)
 from pyomo.dataportal.factory import DataManagerFactory, UnknownDataManager
 
 try:
@@ -33,57 +35,56 @@
     unicode = str
 try:
     long
-    numlist = (bool, int, float, long)
+    numlist = {bool, int, float, long}
 except:
-    numlist = (bool, int, float)
+    numlist = {bool, int, float}
 
 logger = logging.getLogger('pyomo.core')
 
 global Lineno
 global Filename
 
+_num_pattern = re.compile("^("+_re_number+")$")
+_str_false_values = {'False','false','FALSE'}
+_str_bool_values = {'True','true','TRUE'}
+_str_bool_values.update(_str_false_values)
 
 def _process_token(token):
+    #print("TOKEN:", token, type(token))
     if type(token) is tuple:
         return tuple(_process_token(i) for i in token)
-    if type(token) in numlist:
+    elif type(token) in numlist:
         return token
-    if token in ('True','true','TRUE'):
-        return True
-    if token in ('False','false','FALSE'):
-        return False
-
-    if token[0] == '[' and token[-1] == ']':
+    elif token in _str_bool_values:
+        return token not in _str_false_values
+    elif token[0] == '"' and token[-1] == '"':
+        # Strip "flag" quotation characters
+        return token[1:-1]
+    elif token[0] == '[' and token[-1] == ']':
         vals = []
         token = token[1:-1]
         for item in token.split(","):
-            if item[0] == "'" or item[0] == '"':
+            if item[0] in '"\'' and item[0] == item[-1]:
                 vals.append( item[1:-1] )
-            try:
-                vals.append( int(item) )
-                continue
-            except:
-                pass
-            try:
-                vals.append( float(item) )
-                continue
-            except:
-                pass
-            vals.append( item )
+            elif _num_pattern.match(item):
+                _num = float(item)
+                if '.' in item:
+                    vals.append(_num)
+                else:
+                    _int = int(_num)
+                    vals.append(_int if _int == _num else _num)
+            else:
+                vals.append( item )
         return tuple(vals)
-
-    elif token[0] == "'" or token[0] == '"':
-        return token[1:-1]
-
-    try:
-        return int(token)
-    except:
-        pass
-    try:
-        return float(token)
-    except:
-        pass
-    return token
+    elif _num_pattern.match(token):
+        _num = float(token)
+        if '.' in token:
+            return _num
+        else:
+            _int = int(_num)
+            return _int if _int == _num else _num
+    else:
+        return token
 
 
 def _preprocess_data(cmd):


Test patch:
diff --git a/pyomo/dataportal/tests/data_types.dat b/pyomo/dataportal/tests/data_types.dat
new file mode 100644
--- /dev/null
+++ b/pyomo/dataportal/tests/data_types.dat
@@ -0,0 +1,111 @@
+param: I: p :=
+# simple integers
+ 501   2
+ 502  +2
+ 551  -2
+# scientific integers
+ 510   2E2
+ 511   2E+2
+ 512   2e2
+ 513   2e+2
+ 514  +2E2
+ 515  +2E+2
+ 516  +2e2
+ 517  +2e+2
+ 520  -2E2
+ 521  -2E+2
+ 522  -2e2
+ 523  -2e+2
+# scientific (non-integer)
+ 530   2E-2
+ 531   2e-2
+ 532  +2E-2
+ 533  +2e-2
+ 540  -2E-2
+ 541  -2e-2
+# basic floats
+ 100   1.0
+ 101   1.
+ 102  +1.0
+ 103  +1.
+ 110  -1.0
+ 111  -1.
+ 120    .1
+ 121   +.1
+ 130   -.1
+ 140   1.1
+ 141  +1.1
+ 150  -1.1
+# scientific floats
+ 200   2.E2
+ 201   2.E+2
+ 202   2.e2
+ 203   2.e+2
+ 204  +2.E2
+ 205  +2.E+2
+ 206  +2.e2
+ 207  +2.e+2
+ 210  -2.E2
+ 211  -2.E+2
+ 212  -2.e2
+ 213  -2.e+2
+# scientific floats (negative exponents)
+ 220   2.E-2
+ 221   2.e-2
+ 222  +2.E-2
+ 223  +2.e-2
+ 230  -2.E-2
+ 231  -2.e-2
+# scientific floats (with fractional)
+ 300   2.1E2
+ 301   2.1E+2
+ 302   2.1e2
+ 303   2.1e+2
+ 304  +2.1E2
+ 305  +2.1E+2
+ 306  +2.1e2
+ 307  +2.1e+2
+ 310  -2.1E2
+ 311  -2.1E+2
+ 312  -2.1e2
+ 313  -2.1e+2
+# scientific floats (fractional, negative exponents)
+ 320   2.1E-2
+ 321   2.1e-2
+ 322  +2.1E-2
+ 323  +2.1e-2
+ 330  -2.1E-2
+ 331  -2.1e-2
+# scientific floats (with fractional)
+ 400   .1E2
+ 401   .1E+2
+ 402   .1e2
+ 403   .1e+2
+ 404  +.1E2
+ 405  +.1E+2
+ 406  +.1e2
+ 407  +.1e+2
+ 410  -.1E2
+ 411  -.1E+2
+ 412  -.1e2
+ 413  -.1e+2
+/* scientific floats (fractional, negative exponents) */
+ 420   .1E-2
+ 421   .1e-2
+ 422  +.1E-2
+ 423  +.1e-2
+ 430  -.1E-2
+ 431  -.1e-2
+/*Strings*/
+1000  a_string
+1001  "a_string"
+1002  'a_string'
+1003  "a "" string"
+1004  'a '' string'
+1005  1234_567
+1006  "123"
+/* and
+ a
+ multi-line
+ comment*/
+;
diff --git a/pyomo/dataportal/tests/test_dataportal.py b/pyomo/dataportal/tests/test_dataportal.py
--- a/pyomo/dataportal/tests/test_dataportal.py
+++ b/pyomo/dataportal/tests/test_dataportal.py
@@ -428,6 +428,58 @@ def test_md17(self):
         except IOError:
             pass
 
+    def test_dat_type_conversion(self):
+        model = AbstractModel()
+        model.I = Set()
+        model.p = Param(model.I, domain=Any)
+        i = model.create_instance(currdir+"data_types.dat")
+        ref = {
+            50:  (int, 2),
+            55: (int, -2),
+            51:  (int, 200),
+            52: (int, -200),
+            53: (float, 0.02),
+            54: (float, -0.02),
+            10: (float, 1.),
+            11: (float, -1.),
+            12: (float, .1),
+            13: (float, -.1),
+            14: (float, 1.1),
+            15: (float, -1.1),
+            20: (float, 200.),
+            21: (float, -200.),
+            22: (float, .02),
+            23: (float, -.02),
+            30: (float, 210.),
+            31: (float, -210.),
+            32: (float, .021),
+            33: (float, -.021),
+            40: (float, 10.),
+            41: (float, -10.),
+            42: (float, .001),
+            43: (float, -.001),
+            1000: (str, "a_string"),
+            1001: (str, "a_string"),
+            1002: (str, 'a_string'),
+            1003: (str, 'a " string'),
+            1004: (str, "a ' string"),
+            1005: (str, '1234_567'),
+            1006: (str, '123'),
+        }
+        for k, v in i.p.items():
+            #print(k,v, type(v))
+            if k in ref:
+                err="index %s: (%s, %s) does not match ref %s" % (
+                    k, type(v), v, ref[k],)
+                self.assertIs(type(v), ref[k][0], err)
+                self.assertEqual(v, ref[k][1], err)
+            else:
+                n = k // 10
+                err="index %s: (%s, %s) does not match ref %s" % (
+                    k, type(v), v, ref[n],)
+                self.assertIs(type(v), ref[n][0], err)
+                self.assertEqual(v, ref[n][1], err)
+
     def test_data_namespace(self):
         model=AbstractModel()
         model.a=Param()