Repository: ibis-project/ibis

Issue:
feat: warn when limit reached in expr.execute()
When executing an expression a limit is silently applied to what is returned (unless you pass `limit=None`). A user could accidentally continue with local analysis on an arbitrary subset without realizing it's a subset. I'd like an optional parameter/setting that causes it to write a warning to `stderr` or raise an exception if the limit was exceeded. 

Since the limit is currently implemented by adding a limit expression to the original query, it may be difficult to know whether the limit would have been exceeded. In that case, maybe it could just check to see if the number of returned records equals the limit and issue the warning then. It might not definitively mean that the limit would've been exceeded, but would still be useful to get a warning that the limit was potentially exceeded. 


Discussion:
I very much agree with this.
Is there a reason to keep this?
IIRC we removed that for omniscidb backend
To keep what, @xmnlab ?
Do you mean the application of a (default) limit in the execute method?
sorry I was not clear ... I mean .. is there a reason to keep a default limit != None?

we are setting that to None inside `omniscidb` because it was generating conflict with `sql` method at at that time ... after that we change the algorithm related to this method .. and probably it wouldn't be a problem now .. but we kept it as None.

Resolving patch:
diff --git a/ibis/backends/duckdb/registry.py b/ibis/backends/duckdb/registry.py
--- a/ibis/backends/duckdb/registry.py
+++ b/ibis/backends/duckdb/registry.py
@@ -144,6 +144,15 @@ def _regex_extract(t, op):
     return result
 
 
+def _json_get_item(t, op):
+    left, path = map(t.translate, op.args)
+    # Workaround for https://github.com/duckdb/duckdb/issues/5063
+    # In some situations duckdb silently does the wrong thing if
+    # the path is parametrized.
+    sa_path = sa.text(str(path.compile(compile_kwargs=dict(literal_binds=True))))
+    return left.op("->")(sa_path)
+
+
 def _strftime(t, op):
     format_str = op.format_str
     if not isinstance(format_str_op := format_str, ops.Literal):
@@ -220,6 +229,7 @@ def _struct_column(t, op):
         ops.ArgMin: reduction(sa.func.min_by),
         ops.ArgMax: reduction(sa.func.max_by),
         ops.BitwiseXor: fixed_arity(sa.func.xor, 2),
+        ops.JSONGetItem: _json_get_item,
     }
 )
 
diff --git a/ibis/config.py b/ibis/config.py
--- a/ibis/config.py
+++ b/ibis/config.py
@@ -67,7 +67,7 @@ class SQL(Config):
         Dialect to use for printing SQL when the backend cannot be determined.
     """
 
-    default_limit: Optional[PosInt] = 10_000
+    default_limit: Optional[PosInt] = None
     default_dialect: str = "duckdb"
 
 


Test patch:
diff --git a/ibis/backends/sqlite/tests/test_client.py b/ibis/backends/sqlite/tests/test_client.py
--- a/ibis/backends/sqlite/tests/test_client.py
+++ b/ibis/backends/sqlite/tests/test_client.py
@@ -83,10 +83,7 @@ def test_verbose_log_queries(con):
 
     assert len(queries) == 1
     (query,) = queries
-    expected = 'SELECT t0.year \n'
-    expected += 'FROM main.functional_alltypes AS t0\n'
-    expected += ' LIMIT ? OFFSET ?'
-    assert query == expected
+    assert "SELECT t0.year" in query
 
 
 def test_table_equality(dbpath):
diff --git a/ibis/tests/test_config.py b/ibis/tests/test_config.py
--- a/ibis/tests/test_config.py
+++ b/ibis/tests/test_config.py
@@ -4,11 +4,13 @@
 
 
 def test_sql_config():
-    assert options.sql.default_limit == 10_000
+    try:
+        assert options.sql.default_limit is None
 
-    with pytest.raises(TypeError):
-        options.sql.default_limit = -1
+        with pytest.raises(TypeError):
+            options.sql.default_limit = -1
 
-    options.sql.default_limit = 100
-    assert options.sql.default_limit == 100
-    options.sql.default_limit = 10_000
+        options.sql.default_limit = 100
+        assert options.sql.default_limit == 100
+    finally:
+        options.sql.default_limit = None