Repository: ibis-project/ibis

Issue:
feat(clickhouse): implement `.sql`
### What happened?

When I run the following code, I got an error message

`con=ibis.clickhouse.connect()`
`t = con.table('t')`
`t.alias('test').sql('select * from test')`

### What version of ibis are you using?

6.0.0

### What backend(s) are you using, if any?

clickhouse

### Relevant log output

```sh
DatabaseError                             Traceback (most recent call last)
/tmp/ipykernel_30668/2765872492.py in <cell line: 1>()
----> 1 t.alias('test').sql('select * from test limit 5')

~/miniconda3/envs/kenkyu39/lib/python3.9/site-packages/ibis/expr/types/relations.py in sql(self, query, dialect)
   2680             backend = self._find_backend()
   2681             query = backend._transpile_sql(query, dialect=dialect)
-> 2682         op = ops.SQLStringView(child=self, name=next(_ALIASES), query=query)
   2683         return op.to_expr()
   2684 

~/miniconda3/envs/kenkyu39/lib/python3.9/site-packages/ibis/common/grounds.py in __call__(cls, *args, **kwargs)
     23 
     24     def __call__(cls, *args, **kwargs) -> Base:
---> 25         return cls.__create__(*args, **kwargs)
     26 
     27 

~/miniconda3/envs/kenkyu39/lib/python3.9/site-packages/ibis/common/grounds.py in __create__(cls, *args, **kwargs)
     94         # construct the instance by passing the validated keyword arguments
     95         kwargs = cls.__signature__.validate(*args, **kwargs)
---> 96         return super().__create__(**kwargs)
     97 
     98     @classmethod

~/miniconda3/envs/kenkyu39/lib/python3.9/site-packages/ibis/common/grounds.py in __init__(self, **kwargs)
    110         for name, field in self.__attributes__.items():
    111             if isinstance(field, Attribute):
--> 112                 if (value := field.initialize(self)) is not EMPTY:
    113                     object.__setattr__(self, name, value)
    114 

~/miniconda3/envs/kenkyu39/lib/python3.9/site-packages/ibis/common/annotations.py in initialize(self, this)
     83             return EMPTY
     84         elif callable(self._default):
---> 85             value = self._default(this)
     86         else:
     87             value = self._default

~/miniconda3/envs/kenkyu39/lib/python3.9/site-packages/ibis/expr/operations/relations.py in schema(self)
    614         # TODO(kszucs): avoid converting to expression
    615         backend = self.child.to_expr()._find_backend()
--> 616         return backend._get_schema_using_query(self.query)
    617 
    618 

~/miniconda3/envs/kenkyu39/lib/python3.9/site-packages/ibis/backends/clickhouse/__init__.py in _get_schema_using_query(self, query)
    510     def _get_schema_using_query(self, query: str) -> sch.Schema:
    511         query = f"EXPLAIN json = 1, description = 0, header = 1 {query}"
--> 512         with closing(self.raw_sql(query)) as results:
    513             [[raw_plans]] = results.result_columns
    514         [plan] = json.loads(raw_plans)

~/miniconda3/envs/kenkyu39/lib/python3.9/site-packages/ibis/backends/clickhouse/__init__.py in raw_sql(self, query, external_tables, **kwargs)
    469         external_data = self._normalize_external_tables(external_tables)
    470         self._log(query)
--> 471         return self.con.query(query, external_data=external_data, **kwargs)
    472 
    473     def fetch_from_cursor(self, cursor, schema):

~/miniconda3/envs/kenkyu39/lib/python3.9/site-packages/clickhouse_connect/driver/client.py in query(self, query, parameters, settings, query_formats, column_formats, encoding, use_none, column_oriented, use_numpy, max_str_len, context, query_tz, column_tzs, external_data)
    186                                     external_data=query_context.external_data)
    187             return QueryResult([response] if isinstance(response, list) else [[response]])
--> 188         return self._query_with_context(query_context)
    189 
    190     def query_column_block_stream(self,

~/miniconda3/envs/kenkyu39/lib/python3.9/site-packages/clickhouse_connect/driver/httpclient.py in _query_with_context(self, context)
    206             fields = None
    207             headers['Content-Type'] = 'text/plain; charset=utf-8'
--> 208         response = self._raw_request(body,
    209                                      params,
    210                                      headers,

~/miniconda3/envs/kenkyu39/lib/python3.9/site-packages/clickhouse_connect/driver/httpclient.py in _raw_request(self, data, params, headers, method, retries, stream, server_wait, fields, error_handler)
    405                 if error_handler:
    406                     error_handler(response)
--> 407                 self._error_handler(response)
    408 
    409     def ping(self):

~/miniconda3/envs/kenkyu39/lib/python3.9/site-packages/clickhouse_connect/driver/httpclient.py in _error_handler(self, response, retried)
    330             logger.error(err_msg)
    331             err_str = f':{err_str}\n {err_msg[0:240]}'
--> 332         raise OperationalError(err_str) if retried else DatabaseError(err_str) from None
    333 
    334     def _raw_request(self,

DatabaseError: :HTTPDriver returned response code 404)
 Code: 60. DB::Exception: Table prod.test doesn't exist. (UNKNOWN_TABLE) (version 23.4.2.11 (official build))
```


### Code of Conduct

- [X] I agree to follow this project's Code of Conduct


Discussion:
Thanks for the report! I can reproduce this.

It may be a driver issue that needs an upstream fix. I'll report back with details.
Good news: It turns out we just haven't implemented this feature for the ClickHouse backend.
Other news: This will be a challenging feature to implement 😄 
> Good news: It turns out we just haven't implemented this feature for the ClickHouse backend. Other news: This will be a challenging feature to implement 😄

Got it, thanks @cpcloud 

Shall we update the backend operation matrix?
> Shall we update the backend operation matrix?

The backend operation matrix only shows column and scalar operations and the operations that need to be implemented to support `.sql` are table operations.

Is there something in particular you see that is incorrect?
> > Shall we update the backend operation matrix?
> 
> The backend operation matrix only shows column and scalar operations and the operations that need to be implemented to support `.sql` are table operations.
> 
> Is there something in particular you see that is incorrect?

I made the wrong assumption that .alias and .sql always go hand-in-hand and expected .sql to work when I see .alias is implemented for clickhouse.

Thanks for the explanation. It's clear now 

Resolving patch:
diff --git a/ibis/backends/clickhouse/__init__.py b/ibis/backends/clickhouse/__init__.py
--- a/ibis/backends/clickhouse/__init__.py
+++ b/ibis/backends/clickhouse/__init__.py
@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import ast
+import atexit
 import glob
 import json
 from contextlib import closing, suppress
@@ -167,6 +168,7 @@ def do_connect(
             compress=compression,
             **kwargs,
         )
+        self._temp_views = set()
 
     @property
     def version(self) -> str:
@@ -733,3 +735,27 @@ def _load_into_cache(self, name, expr):
 
     def _clean_up_cached_table(self, op):
         self.drop_table(op.name)
+
+    def _create_temp_view(self, table_name, source):
+        if table_name not in self._temp_views and table_name in self.list_tables():
+            raise ValueError(
+                f"{table_name} already exists as a non-temporary table or view"
+            )
+        src = sg.exp.Create(
+            this=sg.table(table_name),  # CREATE ... 'table_name'
+            kind="VIEW",  # VIEW
+            replace=True,  # OR REPLACE
+            expression=source,  # AS ...
+        )
+        self.raw_sql(src.sql(dialect=self.name, pretty=True))
+        self._temp_views.add(table_name)
+        self._register_temp_view_cleanup(table_name)
+
+    def _register_temp_view_cleanup(self, name: str) -> None:
+        def drop(self, name: str, query: str):
+            self.raw_sql(query)
+            self._temp_views.discard(name)
+
+        src = sg.exp.Drop(this=sg.table(name), kind="VIEW", exists=True)
+        query = src.sql(self.name, pretty=True)
+        atexit.register(drop, self, name=name, query=query)
diff --git a/ibis/backends/clickhouse/compiler/relations.py b/ibis/backends/clickhouse/compiler/relations.py
--- a/ibis/backends/clickhouse/compiler/relations.py
+++ b/ibis/backends/clickhouse/compiler/relations.py
@@ -231,3 +231,12 @@ def _dropna(op: ops.DropNa, *, table, how, subset, **_):
 def _sql_string_view(op: ops.SQLStringView, query: str, **_: Any):
     table = sg.table(op.name)
     return sg.select(STAR).from_(table).with_(table, as_=query, dialect="clickhouse")
+
+
+@translate_rel.register
+def _view(op: ops.View, *, child, name: str, **_):
+    # TODO: find a way to do this without creating a temporary view
+    backend = op.child.to_expr()._find_backend()
+    source = sg.select(STAR).from_(child)
+    backend._create_temp_view(table_name=name, source=source)
+    return sg.table(name)


Test patch:
diff --git a/ibis/backends/tests/test_dot_sql.py b/ibis/backends/tests/test_dot_sql.py
--- a/ibis/backends/tests/test_dot_sql.py
+++ b/ibis/backends/tests/test_dot_sql.py
@@ -14,9 +14,7 @@
 except ImportError:
     PolarsComputeError = None
 
-table_dot_sql_notimpl = pytest.mark.notimpl(
-    ["bigquery", "clickhouse", "impala", "druid"]
-)
+table_dot_sql_notimpl = pytest.mark.notimpl(["bigquery", "impala", "druid"])
 dot_sql_notimpl = pytest.mark.notimpl(["datafusion"])
 dot_sql_notyet = pytest.mark.notyet(
     ["snowflake", "oracle"],
@@ -151,11 +149,11 @@ def test_table_dot_sql_with_join(backend, con):
             """
             SELECT
                 l.fancy_af AS yas,
-                r.s
+                r.s AS s
             FROM awesome_t AS l
             LEFT JOIN ft AS r
             ON l.s = r.s
-            """
+            """  # clickhouse needs the r.s AS s, otherwise the column name is returned as r.s
         )
         .order_by(["s", "yas"])
     )
@@ -250,7 +248,6 @@ def test_dot_sql_reuse_alias_with_different_types(backend, alltypes, df):
         *no_sqlglot_dialect,
     ],
 )
-@pytest.mark.broken(["clickhouse"], raises=DatabaseError)
 @pytest.mark.notyet(["polars"], raises=PolarsComputeError)
 @table_dot_sql_notimpl
 @dot_sql_notimpl