Repository: dask/dask

Issue:
Broken usage of capsys for inspecting logging output
<!-- Please include a self-contained copy-pastable example that generates the issue if possible.

Please be concise with code posted. See guidelines below on how to provide a good bug report:

- Craft Minimal Bug Reports http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports
- Minimal Complete Verifiable Examples https://stackoverflow.com/help/mcve

Bug reports that follow these guidelines are easier to diagnose, and so are often handled much more quickly.
-->

**What happened**:

1. We have one unit test that uses capsys to make assertions about emitted logs: [`dask/dataframe/io/tests/test_sql.py`](https://github.com/dask/dask/blob/2022.01.0/dask/dataframe/io/tests/test_sql.py#L410-L425)
1. Add a second unit test that does the same
1. You can't have more than one test using that pattern, the first one works and anything after does not
    - add new test after old test: old test passes, new test fails
    - skip old test: new test passes
    - add new test before old test: new test passes, old test fails

**What you expected to happen**:

We can have as many tests as we want making assertions about logs.

**Minimal Complete Verifiable Example**:

```python
# repro.py
import sqlalchemy as sa
from tempfile import NamedTemporaryFile

def do_something():
    with NamedTemporaryFile() as f:
        engine = sa.create_engine(f'sqlite:///{f.name}', echo=True)
        engine.execute("CREATE TABLE t (id INTEGER);")

def test1(capsys):
    do_something()
    out, err = capsys.readouterr()
    assert 'CREATE' in out

def test2(capsys):
    do_something()
    out, err = capsys.readouterr()
    assert 'CREATE' in out
```

Usage:
```sh
$ pytest -v repro.py -k test1 # passes
$ pytest -v repro.py -k test2 # passes
$ pytest -v repro.py # test1 passes, test2 fails
```

<details> <summary> inspected stderr of failed test case </summary>
<pre>
Traceback (most recent call last):
  File "/usr/lib/python3.8/logging/__init__.py", line 1088, in emit
    stream.write(msg + self.terminator)
ValueError: I/O operation on closed file.
Call stack:
  File "/dask/venv/bin/pytest", line 8, in <module>
    sys.exit(console_main())
  File "/dask/venv/lib/python3.8/site-packages/_pytest/config/__init__.py", line 185, in console_main
    code = main()
  File "/dask/venv/lib/python3.8/site-packages/_pytest/config/__init__.py", line 162, in main
    ret: Union[ExitCode, int] = config.hook.pytest_cmdline_main(
  File "/dask/venv/lib/python3.8/site-packages/pluggy/_hooks.py", line 265, in __call__
    return self._hookexec(self.name, self.get_hookimpls(), kwargs, firstresult)
  File "/dask/venv/lib/python3.8/site-packages/pluggy/_manager.py", line 80, in _hookexec
    return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
  File "/dask/venv/lib/python3.8/site-packages/pluggy/_callers.py", line 39, in _multicall
    res = hook_impl.function(*args)
  File "/dask/venv/lib/python3.8/site-packages/_pytest/main.py", line 316, in pytest_cmdline_main
    return wrap_session(config, _main)
  File "/dask/venv/lib/python3.8/site-packages/_pytest/main.py", line 269, in wrap_session
    session.exitstatus = doit(config, session) or 0
  File "/dask/venv/lib/python3.8/site-packages/_pytest/main.py", line 323, in _main
    config.hook.pytest_runtestloop(session=session)
  File "/dask/venv/lib/python3.8/site-packages/pluggy/_hooks.py", line 265, in __call__
    return self._hookexec(self.name, self.get_hookimpls(), kwargs, firstresult)
  File "/dask/venv/lib/python3.8/site-packages/pluggy/_manager.py", line 80, in _hookexec
    return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
  File "/dask/venv/lib/python3.8/site-packages/pluggy/_callers.py", line 39, in _multicall
    res = hook_impl.function(*args)
  File "/dask/venv/lib/python3.8/site-packages/_pytest/main.py", line 348, in pytest_runtestloop
    item.config.hook.pytest_runtest_protocol(item=item, nextitem=nextitem)
  File "/dask/venv/lib/python3.8/site-packages/pluggy/_hooks.py", line 265, in __call__
    return self._hookexec(self.name, self.get_hookimpls(), kwargs, firstresult)
  File "/dask/venv/lib/python3.8/site-packages/pluggy/_manager.py", line 80, in _hookexec
    return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
  File "/dask/venv/lib/python3.8/site-packages/pluggy/_callers.py", line 39, in _multicall
    res = hook_impl.function(*args)
  File "/dask/venv/lib/python3.8/site-packages/_pytest/runner.py", line 109, in pytest_runtest_protocol
    runtestprotocol(item, nextitem=nextitem)
  File "/dask/venv/lib/python3.8/site-packages/_pytest/runner.py", line 126, in runtestprotocol
    reports.append(call_and_report(item, "call", log))
  File "/dask/venv/lib/python3.8/site-packages/_pytest/runner.py", line 215, in call_and_report
    call = call_runtest_hook(item, when, **kwds)
  File "/dask/venv/lib/python3.8/site-packages/_pytest/runner.py", line 254, in call_runtest_hook
    return CallInfo.from_call(
  File "/dask/venv/lib/python3.8/site-packages/_pytest/runner.py", line 311, in from_call
    result: Optional[TResult] = func()
  File "/dask/venv/lib/python3.8/site-packages/_pytest/runner.py", line 255, in <lambda>
    lambda: ihook(item=item, **kwds), when=when, reraise=reraise
  File "/dask/venv/lib/python3.8/site-packages/pluggy/_hooks.py", line 265, in __call__
    return self._hookexec(self.name, self.get_hookimpls(), kwargs, firstresult)
  File "/dask/venv/lib/python3.8/site-packages/pluggy/_manager.py", line 80, in _hookexec
    return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
  File "/dask/venv/lib/python3.8/site-packages/pluggy/_callers.py", line 39, in _multicall
    res = hook_impl.function(*args)
  File "/dask/venv/lib/python3.8/site-packages/_pytest/runner.py", line 162, in pytest_runtest_call
    item.runtest()
  File "/dask/venv/lib/python3.8/site-packages/_pytest/python.py", line 1641, in runtest
    self.ihook.pytest_pyfunc_call(pyfuncitem=self)
  File "/dask/venv/lib/python3.8/site-packages/pluggy/_hooks.py", line 265, in __call__
    return self._hookexec(self.name, self.get_hookimpls(), kwargs, firstresult)
  File "/dask/venv/lib/python3.8/site-packages/pluggy/_manager.py", line 80, in _hookexec
    return self._inner_hookexec(hook_name, methods, kwargs, firstresult)
  File "/dask/venv/lib/python3.8/site-packages/pluggy/_callers.py", line 39, in _multicall
    res = hook_impl.function(*args)
  File "/dask/venv/lib/python3.8/site-packages/_pytest/python.py", line 183, in pytest_pyfunc_call
    result = testfunction(**testargs)
  File "/dask/repro.py", line 23, in test2
    do_something()
  File "/dask/repro.py", line 13, in do_something
    engine.execute("CREATE TABLE t (id INTEGER);")
  File "<string>", line 2, in execute
  File "/dask/venv/lib/python3.8/site-packages/sqlalchemy/util/deprecations.py", line 401, in warned
    return fn(*args, **kwargs)
  File "/dask/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 3146, in execute
    return connection.execute(statement, *multiparams, **params)
  File "/dask/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1274, in execute
    return self._exec_driver_sql(
  File "/dask/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1578, in _exec_driver_sql
    ret = self._execute_context(
  File "/dask/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 1755, in _execute_context
    self._log_info(statement)
  File "/dask/venv/lib/python3.8/site-packages/sqlalchemy/engine/base.py", line 136, in _log_info
    self.engine.logger.info(message, *arg, **kw)
  File "/dask/venv/lib/python3.8/site-packages/sqlalchemy/log.py", line 117, in info
    self.log(logging.INFO, msg, *args, **kwargs)
  File "/dask/venv/lib/python3.8/site-packages/sqlalchemy/log.py", line 163, in log
    self.logger._log(level, msg, args, **kwargs)
Message: 'CREATE TABLE t (id INTEGER);'
</pre>
</details>

**Anything else we need to know?**: the issue manifested in #8609.

**Environment**:

- SQLAlchemy verison: 1.4.31
- pytest version: 6.2.5
- Dask version: main
- Python version: 3.8.10
- Operating System: Ubuntu 20.04
- Install method (conda, pip, source): source


Discussion:

Resolving patch:
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -1572,6 +1572,7 @@ def to_sql(
         method=None,
         compute=True,
         parallel=False,
+        engine_kwargs=None,
     ):
         """See dd.to_sql docstring for more information"""
         from .io import to_sql
@@ -1589,6 +1590,7 @@ def to_sql(
             method=method,
             compute=compute,
             parallel=parallel,
+            engine_kwargs=engine_kwargs,
         )
 
     def to_json(self, filename, *args, **kwargs):
diff --git a/dask/dataframe/io/sql.py b/dask/dataframe/io/sql.py
--- a/dask/dataframe/io/sql.py
+++ b/dask/dataframe/io/sql.py
@@ -678,6 +678,7 @@ def to_sql(
     method=None,
     compute=True,
     parallel=False,
+    engine_kwargs=None,
 ):
     """Store Dask Dataframe to a SQL table
 
@@ -734,6 +735,8 @@ def to_sql(
         When true, have each block append itself to the DB table concurrently. This can result in DB rows being in a
         different order than the source DataFrame's corresponding rows. When false, load each block into the SQL DB in
         sequence.
+    engine_kwargs : dict or None
+        Specific db engine parameters for sqlalchemy
 
     Raises
     ------
@@ -786,13 +789,18 @@ def to_sql(
     >>> result
     [(0, 0, '00'), (1, 1, '11'), (2, 2, '22'), (3, 3, '33')]
     """
+    import sqlalchemy as sa
+
     if not isinstance(uri, str):
         raise ValueError(f"Expected URI to be a string, got {type(uri)}.")
 
+    engine_kwargs = {} if engine_kwargs is None else engine_kwargs
+    engine = sa.create_engine(uri, **engine_kwargs)
+
     # This is the only argument we add on top of what Pandas supports
     kwargs = dict(
         name=name,
-        con=uri,
+        con=engine,
         schema=schema,
         if_exists=if_exists,
         index=index,


Test patch:
diff --git a/dask/dataframe/io/tests/test_sql.py b/dask/dataframe/io/tests/test_sql.py
--- a/dask/dataframe/io/tests/test_sql.py
+++ b/dask/dataframe/io/tests/test_sql.py
@@ -361,19 +361,19 @@ def test_datetimes():
         assert_eq(data.map_partitions(lambda x: x.sort_index()), df2.sort_index())
 
 
-def test_extra_connection_engine_keywords(capsys, db):
+def test_extra_connection_engine_keywords(caplog, db):
     data = read_sql_table(
         "test", db, npartitions=2, index_col="number", engine_kwargs={"echo": False}
     ).compute()
     # no captured message from the stdout with the echo=False parameter (this is the default)
-    out, err = capsys.readouterr()
-    assert "SELECT" not in out
+    out = "\n".join(r.message for r in caplog.records)
+    assert out == ""
     assert_eq(data, df)
     # with the echo=True sqlalchemy parameter, you should get all SQL queries in the stdout
     data = read_sql_table(
         "test", db, npartitions=2, index_col="number", engine_kwargs={"echo": True}
     ).compute()
-    out, err = capsys.readouterr()
+    out = "\n".join(r.message for r in caplog.records)
     assert "WHERE" in out
     assert "FROM" in out
     assert "SELECT" in out
@@ -552,3 +552,20 @@ def test_to_sql_kwargs():
             TypeError, match="to_sql\\(\\) got an unexpected keyword argument 'unknown'"
         ):
             ddf.to_sql("test", uri, unknown=None)
+
+
+def test_to_sql_engine_kwargs(caplog):
+    ddf = dd.from_pandas(df, 2)
+    with tmp_db_uri() as uri:
+        ddf.to_sql("test", uri, engine_kwargs={"echo": False})
+        logs = "\n".join(r.message for r in caplog.records)
+        assert logs == ""
+        assert_eq(df, read_sql_table("test", uri, "number"))
+
+    with tmp_db_uri() as uri:
+        ddf.to_sql("test", uri, engine_kwargs={"echo": True})
+        logs = "\n".join(r.message for r in caplog.records)
+        assert "CREATE" in logs
+        assert "INSERT" in logs
+
+        assert_eq(df, read_sql_table("test", uri, "number"))