Repository: Pyomo/pyomo

Issue:
Obtain column names from `cursor.description()` in sql query
Dear Pyomo team:

I came across legacy issue [#4480](https://software.sandia.gov/trac/pyomo/ticket/4480) and I would like to ask whether it is possible to "revive" it. I have tested and had success with the suggestion proposed in the legacy ticket of removing the code that extracts column names from the data file query, or fetches them from the database with a `SELECT *` type statement. I have removed that code and replaced it thus with a statement that obtains column names from the cursor. The portion of the code in question in `core/plugins/data/db_read.py` (for Pyomo 5.5.0) reads as follows:
```python
def read(self):
        #
        # Get the table from the database
        #
        if self.db is None:
            return
        cursor = self.db.cursor()
        tmp = []
        if self.options.query is None:
            if self.options.table is None:
                raise IOError("Must specify 'query' or 'table' option!")
            self.options.query = 'SELECT * FROM %s' % self.options.table
        elif self.options.query[0] in ("'", '"'):
            self.options.query = self.options.query[1:-1]
        cursor.execute(self.options.query)
        try:
            rows = cursor.fetchall()
            for col in cursor.description:
                tmp.append(col[0])
            tmp=[tmp]
        except: # subsequent code remains the same in db_read.py
```


The reason for this request is that I do not have control over the name of the columns in the database I need to access, and renaming sets and variables in pyomo to match results in code that is less readable.

In fact when using pyomo with the modified functionality as above, 
I am able to simply put an alias on all the columns I need. 
For example: a query to query for the value of a parameter `P` that is indexed over 3 sets `A`. `B` and `C` would read:
```sql
SELECT database_column_name1 as A, database_column_name2 as B, 
database_column_name3 as C,database_column_name4 as P
FROM dabatase_table
WHERE ... 
```
 
Best regards


Discussion:
@jotaigna if you have a solution to the legacy issue you mentioned, we invite you to open a pull request with your code changes and the Pyomo team will review it and get it merged in. 

Resolving patch:
diff --git a/pyomo/core/data/TableData.py b/pyomo/core/data/TableData.py
--- a/pyomo/core/data/TableData.py
+++ b/pyomo/core/data/TableData.py
@@ -2,8 +2,8 @@
 #
 #  Pyomo: Python Optimization Modeling Objects
 #  Copyright 2017 National Technology and Engineering Solutions of Sandia, LLC
-#  Under the terms of Contract DE-NA0003525 with National Technology and 
-#  Engineering Solutions of Sandia, LLC, the U.S. Government retains certain 
+#  Under the terms of Contract DE-NA0003525 with National Technology and
+#  Engineering Solutions of Sandia, LLC, the U.S. Government retains certain
 #  rights in this software.
 #  This software is distributed under the 3-clause BSD License.
 #  ___________________________________________________________________________
@@ -120,7 +120,11 @@ def _set_data(self, headers, rows):
                 header_index.append(i)
         else:
             for i in self.options.select:
-                header_index.append(headers.index(str(i)))
+                try:
+                    header_index.append(headers.index(str(i)))
+                except:
+                    print("Model declaration '%s' not found in returned query columns" %str(i))
+                    raise
         self.options.ncolumns = len(headers)
 
         if not self.options.param is None:
@@ -268,4 +272,3 @@ def _get_table(self):
                     cols.append(param)
                 tmp = [cols] + tmp
         return tmp
-
diff --git a/pyomo/core/plugins/data/db_table.py b/pyomo/core/plugins/data/db_table.py
--- a/pyomo/core/plugins/data/db_table.py
+++ b/pyomo/core/plugins/data/db_table.py
@@ -2,8 +2,8 @@
 #
 #  Pyomo: Python Optimization Modeling Objects
 #  Copyright 2017 National Technology and Engineering Solutions of Sandia, LLC
-#  Under the terms of Contract DE-NA0003525 with National Technology and 
-#  Engineering Solutions of Sandia, LLC, the U.S. Government retains certain 
+#  Under the terms of Contract DE-NA0003525 with National Technology and
+#  Engineering Solutions of Sandia, LLC, the U.S. Government retains certain
 #  rights in this software.
 #  This software is distributed under the 3-clause BSD License.
 #  ___________________________________________________________________________
@@ -94,44 +94,18 @@ def read(self):
             self.options.query = 'SELECT * FROM %s' % self.options.table
         elif self.options.query[0] in ("'", '"'):
             self.options.query = self.options.query[1:-1]
-        if not self.options.table is None:
-            try:
-                for row in cursor.columns(table=self.options.table):
-                    tmp.append(row.column_name)
-                tmp=[tmp]
-            except:
-                #
-                # TODO: is this only for SQLite?
-                #
-                cursor.execute("SELECT * FROM %s" % self.options.table)
-                for col in cursor.description:
-                    tmp.append(col[0])
-                tmp=[tmp]
-        else:
-            # TODO: extend this logic to create a header row for a SQL query
-            # FIXME: the current regex logic is pretty brittle...
-            match = re.search("SELECT (.*) FROM.*", self.options.query)
-            if match is not None:
-                fieldstr = match.group(1)
-                if fieldstr == "*":
-                    raise ValueError("Couldn't extract field names from query. Please specify database columns explicitly in query.")
-                else:
-                    fields = [f.strip() for f in fieldstr.split(",")]
-                #print "FOUND FIELDS", fields # XXX
-                tmp = [fields]
-                #self.options.index_name = fields[0]
-                #self.options.select = fields[1:]
-            else:
-                # TODO couldn't figure out field names from query; need another strategy!
-                raise ValueError("Couldn't extract field names from query. Please specify database columns explicitly in query.")
-        cursor.execute(self.options.query)
+
         try:
+            cursor.execute(self.options.query)
             rows = cursor.fetchall()
-        except:
+            for col in cursor.description:
+                tmp.append(col[0])
+            tmp = [tmp]
+        except sqlite3.OperationalError:
             import logging
             logging.getLogger('pyomo.core').error(
                 """Fatal error reading from an external ODBC data source.
-                
+
 This error was generated outside Pyomo by the Python connector to the
 external data source:
 
@@ -236,7 +210,7 @@ def connect(self, connection, options):
         try:
             conn = db_Table.connect(self, connection, options, extras)
         except TypeError:
-            raise 
+            raise
         except Exception:
             e = sys.exc_info()[1]
             code = e.args[0]
@@ -381,7 +355,7 @@ def load(self, filename=None, data=None):
                 sections.update(self._get_sections(fileData))
         if data is not None:
             sections.update(self._get_sections(data))
-        
+
         if self.ODBC_DS_KEY in sections:
             self.sources.update(sections[self.ODBC_DS_KEY])
             del sections[self.ODBC_DS_KEY]
@@ -407,7 +381,7 @@ def odbc_repr(self):
         """
 
         str = "[{0}]\n".format(self.ODBC_DS_KEY)
-        
+
         for name in self.sources:
             str += "{0} = {1}\n".format(name, self.sources[name])
 
@@ -475,7 +449,7 @@ def add_source_spec(self, name, spec):
             raise ODBCError("A source spec must have a corresponding source; call .add_source() first")
 
         self.source_specs[name] = dict(spec)
-    
+
     def del_source_spec(self, name):
         """
         Remove an ODBC data source specification from the
@@ -564,7 +538,7 @@ def requirements(self):
 
     def connect(self, connection, options):
         assert(options['using'] == 'pypyodbc')
-        
+
         return pyodbc_db_Table.connect(self, connection, options)
 
 
@@ -584,7 +558,7 @@ def requirements(self):
 
     def connect(self, connection, options):
         assert(options['using'] == 'sqlite3')
-        
+
         filename = connection
         if not os.path.exists(filename):
             raise Exception("No such file: " + filename)
@@ -608,4 +582,3 @@ def available(self):
 
     def requirements(self):
         return 'pymysql'
-


Test patch: