Repository: ibis-project/ibis

Issue:
Implement .insert() for SQLite
Some backends implement the `.insert()` method, to insert rows into tables. SQLite doesn't, and besides being useful for users, it'd be good to have it working, and add an example in the tutorial 5, so it's useful as a reference for other backends too.


Discussion:

Resolving patch:
diff --git a/ibis/backends/base/sql/alchemy/client.py b/ibis/backends/base/sql/alchemy/client.py
--- a/ibis/backends/base/sql/alchemy/client.py
+++ b/ibis/backends/base/sql/alchemy/client.py
@@ -1,6 +1,6 @@
 import contextlib
 import functools
-from typing import List, Optional
+from typing import List, Optional, Union
 
 import pandas as pd
 import sqlalchemy as sa
@@ -9,6 +9,7 @@
 import ibis
 import ibis.expr.datatypes as dt
 import ibis.expr.schema as sch
+import ibis.expr.types as ir
 import ibis.util as util
 from ibis.backends.base.sql.compiler import Dialect
 from ibis.client import Query, SQLClient
@@ -229,7 +230,7 @@ def load_data(
         Parameters
         ----------
         table_name : string
-        data: pandas.DataFrame
+        data : pandas.DataFrame
         database : string, optional
         if_exists : string, optional, default 'fail'
             The values available are: {‘fail’, ‘replace’, ‘append’}
@@ -329,3 +330,93 @@ def _sqla_table_to_expr(self, table):
     def version(self):
         vstring = '.'.join(map(str, self.con.dialect.server_version_info))
         return parse_version(vstring)
+
+    def insert(
+        self,
+        table_name: str,
+        obj: Union[pd.DataFrame, ir.TableExpr],
+        database: Optional[str] = None,
+        overwrite: Optional[bool] = False,
+    ) -> None:
+        """
+        Insert the given data to a table in backend.
+
+        Parameters
+        ----------
+        table_name : string
+            name of the table to which data needs to be inserted
+        obj : pandas DataFrame or ibis TableExpr
+            obj is either the dataframe (pd.DataFrame) containing data
+            which needs to be inserted to table_name or
+            the TableExpr type which ibis provides with data which needs
+            to be inserted to table_name
+        database : string, optional
+            name of the attached database that the table is located in.
+        overwrite : boolean, default False
+            If True, will replace existing contents of table else not
+
+        Raises
+        -------
+        NotImplementedError
+            Inserting data to a table from a different database is not
+            yet implemented
+
+        ValueError
+            No operation is being performed. Either the obj parameter
+            is not a pandas DataFrame or is not a ibis TableExpr.
+            The given obj is of type type(obj).__name__ .
+
+        """
+
+        if database == self.database_name:
+            # avoid fully qualified name
+            database = None
+
+        if database is not None:
+            raise NotImplementedError(
+                'Inserting data to a table from a different database is not '
+                'yet implemented'
+            )
+
+        params = {}
+        if self.has_attachment:
+            # for database with attachment
+            # see: https://github.com/ibis-project/ibis/issues/1930
+            params['schema'] = self.database_name
+
+        if isinstance(obj, pd.DataFrame):
+            obj.to_sql(
+                table_name,
+                self.con,
+                index=False,
+                if_exists='replace' if overwrite else 'append',
+                **params,
+            )
+        elif isinstance(obj, ir.TableExpr):
+            to_table_expr = self.table(table_name)
+            to_table_schema = to_table_expr.schema()
+
+            if overwrite:
+                self.drop_table(table_name, database=database)
+                self.create_table(
+                    table_name, schema=to_table_schema, database=database,
+                )
+
+            to_table = self._get_sqla_table(table_name, schema=database)
+
+            from_table_expr = obj
+
+            with self.begin() as bind:
+                if from_table_expr is not None:
+                    bind.execute(
+                        to_table.insert().from_select(
+                            list(from_table_expr.columns),
+                            from_table_expr.compile(),
+                        )
+                    )
+        else:
+            raise ValueError(
+                "No operation is being performed. Either the obj parameter "
+                "is not a pandas DataFrame or is not a ibis TableExpr."
+                f"The given obj is of type {type(obj).__name__} ."
+            )


Test patch:
diff --git a/ibis/backends/tests/conftest.py b/ibis/backends/tests/conftest.py
--- a/ibis/backends/tests/conftest.py
+++ b/ibis/backends/tests/conftest.py
@@ -3,6 +3,7 @@
 from pathlib import Path
 from typing import List
 
+import pandas as pd
 import pytest
 
 import ibis
@@ -306,3 +307,45 @@ def alternate_current_database(con, backend, current_data_db: str) -> str:
     finally:
         con.set_database(current_data_db)
         con.drop_database(name, force=True)
+
+
+@pytest.fixture
+def test_employee_schema() -> ibis.schema:
+    sch = ibis.schema(
+        [
+            ('first_name', 'string'),
+            ('last_name', 'string'),
+            ('department_name', 'string'),
+            ('salary', 'float64'),
+        ]
+    )
+
+    return sch
+
+
+@pytest.fixture
+def test_employee_data_1():
+    df = pd.DataFrame(
+        {
+            'first_name': ['A', 'B', 'C'],
+            'last_name': ['D', 'E', 'F'],
+            'department_name': ['AA', 'BB', 'CC'],
+            'salary': [100.0, 200.0, 300.0],
+        }
+    )
+
+    return df
+
+
+@pytest.fixture
+def test_employee_data_2():
+    df2 = pd.DataFrame(
+        {
+            'first_name': ['X', 'Y', 'Z'],
+            'last_name': ['A', 'B', 'C'],
+            'department_name': ['XX', 'YY', 'ZZ'],
+            'salary': [400.0, 500.0, 600.0],
+        }
+    )
+
+    return df2
diff --git a/ibis/backends/tests/test_client.py b/ibis/backends/tests/test_client.py
--- a/ibis/backends/tests/test_client.py
+++ b/ibis/backends/tests/test_client.py
@@ -1,10 +1,13 @@
 import pandas as pd
+import pandas.testing as tm
 import pytest
 from pkg_resources import parse_version
 
 import ibis
 import ibis.expr.datatypes as dt
 
+SQLALCHEMY_BACKENDS = ['sqlite', 'postgres', 'mysql']
+
 
 @pytest.fixture
 def new_schema():
@@ -202,3 +205,96 @@ def test_separate_database(con, alternate_current_database, current_data_db):
     db = con.database(current_data_db)
     assert db.name == current_data_db
     assert tmp_db.name == alternate_current_database
+
+
+def _create_temp_table_with_schema(con, temp_table_name, schema, data=None):
+
+    con.drop_table(temp_table_name, force=True)
+    con.create_table(temp_table_name, schema=schema)
+    temporary = con.table(temp_table_name)
+    assert len(temporary.execute()) == 0
+
+    if data is not None and isinstance(data, pd.DataFrame):
+        con.load_data(temp_table_name, data, if_exists='append')
+        assert len(temporary.execute()) == len(data.index)
+        tm.assert_frame_equal(temporary.execute(), data)
+
+    return temporary
+
+
+@pytest.mark.only_on_backends(
+    SQLALCHEMY_BACKENDS, reason="run only if backend is SQLAlchemy based",
+)
+def test_insert_no_overwrite_from_dataframe(
+    con, test_employee_schema, test_employee_data_2
+):
+
+    temp_table = 'temp_to_table'
+    temporary = _create_temp_table_with_schema(
+        con, temp_table, test_employee_schema,
+    )
+
+    con.insert(temp_table, obj=test_employee_data_2, overwrite=False)
+    assert len(temporary.execute()) == 3
+    tm.assert_frame_equal(temporary.execute(), test_employee_data_2)
+
+
+@pytest.mark.only_on_backends(
+    SQLALCHEMY_BACKENDS, reason="run only if backend is SQLAlchemy based",
+)
+def test_insert_overwrite_from_dataframe(
+    con, test_employee_schema, test_employee_data_1, test_employee_data_2
+):
+
+    temp_table = 'temp_to_table'
+    temporary = _create_temp_table_with_schema(
+        con, temp_table, test_employee_schema, data=test_employee_data_1,
+    )
+
+    con.insert(temp_table, obj=test_employee_data_2, overwrite=True)
+    assert len(temporary.execute()) == 3
+    tm.assert_frame_equal(temporary.execute(), test_employee_data_2)
+
+
+@pytest.mark.only_on_backends(
+    SQLALCHEMY_BACKENDS, reason="run only if backend is SQLAlchemy based",
+)
+def test_insert_no_overwite_from_expr(
+    con, test_employee_schema, test_employee_data_2
+):
+
+    temp_table = 'temp_to_table'
+    temporary = _create_temp_table_with_schema(
+        con, temp_table, test_employee_schema,
+    )
+
+    from_table_name = 'temp_from_table'
+    from_table = _create_temp_table_with_schema(
+        con, from_table_name, test_employee_schema, data=test_employee_data_2,
+    )
+
+    con.insert(temp_table, obj=from_table, overwrite=False)
+    assert len(temporary.execute()) == 3
+    tm.assert_frame_equal(temporary.execute(), from_table.execute())
+
+
+@pytest.mark.only_on_backends(
+    SQLALCHEMY_BACKENDS, reason="run only if backend is SQLAlchemy based",
+)
+def test_insert_overwrite_from_expr(
+    con, test_employee_schema, test_employee_data_1, test_employee_data_2
+):
+
+    temp_table = 'temp_to_table'
+    temporary = _create_temp_table_with_schema(
+        con, temp_table, test_employee_schema, data=test_employee_data_1,
+    )
+
+    from_table_name = 'temp_from_table'
+    from_table = _create_temp_table_with_schema(
+        con, from_table_name, test_employee_schema, data=test_employee_data_2,
+    )
+
+    con.insert(temp_table, obj=from_table, overwrite=True)
+    assert len(temporary.execute()) == 3
+    tm.assert_frame_equal(temporary.execute(), from_table.execute())