Repository: ibis-project/ibis

Issue:
(Enhancement) Use a sql query as an expression
Is there a way for ibis to have an expression simply be a static, pre-existing SQL query? e.g. today, the primary workflow for ibis (in our omnisci/mapd backend) is to generate sql via the compilation path, and i get that the primary use case is as a deferred expression system, e.g. the typical flow is

```
conn = ibis.mapd.conn(....)
t = conn.table('tbl')
expr = t[t.colA > 50].group_by(t.colB).aggregate(t.count())...
output_df = expr.execute()
```

however, there are several cases where we find ourselves wanting a pre-existing SQL query to be the 'basis' of an expression - e.g.

```
query_str = 'select a, b, count(*) as ct from tbl where ... group by a,b having ...'
expr_query = expr_from_query_string(query) #(A 'pre-compiled' expression
expr2 = expr_query[(ct > 50) & (a == 'foo)] 
```
ideally should generate a query with a [CTE ](https://www.geeksforgeeks.org/cte-in-sql/) for query_str i.e the above when compiled should return something like
```
WITH t0 as (
select a, b, count(*) as ct from tbl where ... group by a,b having ...'
) select * from t0 where t0.a == 'foo' and t0.ct > 50
```


Discussion:
not sure if it is what `SQLClient.sql` does. I am going to investigate that.

Resolving patch:
diff --git a/ibis/mapd/client.py b/ibis/mapd/client.py
--- a/ibis/mapd/client.py
+++ b/ibis/mapd/client.py
@@ -4,6 +4,7 @@
 import pymapd
 
 from pymapd.cursor import Cursor
+from pymapd._parsers import _extract_column_details
 
 try:
     from cudf.dataframe.dataframe import DataFrame as GPUDataFrame
@@ -12,6 +13,7 @@
 
 import ibis.common as com
 import ibis.expr.datatypes as dt
+import ibis.expr.operations as ops
 import ibis.expr.schema as sch
 import ibis.expr.types as ir
 
@@ -73,6 +75,31 @@ class MapDDataType:
         v: k for k, v in dtypes.items()
     }
 
+    _mapd_to_ibis_dtypes = {
+        'BIGINT': 'int64',
+        'BOOLEAN': 'Boolean',
+        'BOOL': 'Boolean',
+        'CHAR': 'string',
+        'DATE': 'date',
+        'DECIMAL': 'decimal',
+        'DOUBLE': 'double',
+        'INT': 'int32',
+        'INTEGER': 'int32',
+        'FLOAT': 'float32',
+        'NUMERIC': 'float64',
+        'REAL': 'float32',
+        'SMALLINT': 'int16',
+        'STR': 'string',
+        'TEXT': 'string',
+        'TIME': 'time',
+        'TIMESTAMP': 'timestamp',
+        'VARCHAR': 'string',
+        'POINT': 'point',
+        'LINESTRING': 'linestring',
+        'POLYGON': 'polygon',
+        'MULTIPOLYGON': 'multipolygon'
+    }
+
     def __init__(self, typename, nullable=True):
         if typename not in self.dtypes:
             raise com.UnsupportedBackendType(typename)
@@ -355,6 +382,14 @@ def close(self):
         """Close MapD connection and drop any temporary objects"""
         self.con.close()
 
+    def _adapt_types(self, descr):
+        names = []
+        adapted_types = []
+        for col in descr:
+            names.append(col.name)
+            adapted_types.append(MapDDataType._mapd_to_ibis_dtypes[col.type])
+        return names, adapted_types
+
     def _build_ast(self, expr, context):
         """
         Required.
@@ -370,6 +405,18 @@ def _fully_qualified_name(self, name, database):
         # MapD raises error sometimes with qualified names
         return name
 
+    def _get_schema_using_query(self, query):
+        with self._execute(query, results=True) as result:
+            # resets the state of the cursor and closes operation
+            result.cursor.fetchall()
+            names, ibis_types = self._adapt_types(
+                _extract_column_details(
+                    result.cursor._result.row_set.row_desc
+                )
+            )
+
+        return sch.Schema(names, ibis_types)
+
     def _get_table_schema(self, table_name, database=None):
         """
 
@@ -761,6 +808,23 @@ def get_schema(self, table_name, database=None):
             for col in self.con.get_table_details(table_name)
         ])
 
+    def sql(self, query):
+        """
+        Convert a SQL query to an Ibis table expression
+
+        Parameters
+        ----------
+
+        Returns
+        -------
+        table : TableExpr
+        """
+        # Get the schema by adding a LIMIT 0 on to the end of the query. If
+        # there is already a limit in the query, we find and remove it
+        limited_query = 'SELECT * FROM ({}) t0 LIMIT 1'.format(query)
+        schema = self._get_schema_using_query(limited_query)
+        return ops.SQLQueryResult(query, schema, self).to_expr()
+
     @property
     def version(self):
         # pymapd doesn't have __version__


Test patch:
diff --git a/ibis/tests/all/test_client.py b/ibis/tests/all/test_client.py
--- a/ibis/tests/all/test_client.py
+++ b/ibis/tests/all/test_client.py
@@ -1,9 +1,10 @@
+from pkg_resources import parse_version
+
 import pytest
 
 import ibis
 import ibis.tests.util as tu
 import ibis.expr.datatypes as dt
-from pkg_resources import parse_version
 
 
 @tu.skipif_unsupported
@@ -38,3 +39,22 @@ def test_query_schema(backend, con, alltypes, expr_fn, expected):
     expected = ibis.schema([(name, dtype(nullable=schema[name].nullable))
                             for name, dtype in expected])
     assert query.schema().equals(expected)
+
+
+@pytest.mark.parametrize('field', [
+    'bool_col', 'tinyint_col', 'smallint_col',
+    'int_col', 'bigint_col', 'float_col', 'double_col',
+    'date_string_col', 'string_col', 'timestamp_col'
+])
+@tu.skipif_unsupported
+def test_sql(backend, con, alltypes, field):
+
+    if (
+        not hasattr(backend, 'sql')
+        or not hasattr(backend, '_get_schema_using_query')
+    ):
+        pytest.skip('Backend {} does not support sql method'.format(backend))
+
+    result = con.sql(alltypes.compile())
+    assert hasattr(result, field)
+    assert hasattr(alltypes, field)