Repository: pandas-dev/pandas

Issue:
to_sql method turns datetime64 index to time zone aware in postgres
I have a `pandas` `Dataframe` with an index which is of type `datetime64[ns]`.
when I use the `to_sql` method i expect the index to be created as a `timestamp` postgres column, however it is creates a `timestamptz` column.

Note that when the `datetime64[ns]` column is not an index but rather a normal column, this doesn't happen.

pandas 0.23.4, psycopg2 2.7.4, sqlalchemy 1.2.7, PostgreSQL 9.6.6

---
*example*

```python
dates = pd.date_range('2018-01-01', periods=5, freq='6h')
df_test = pd.DataFrame({'nums': range(5)}, index=dates)
```
<details>

|                     | nums |
|---------------------|------|
| 2018-01-01 00:00:00 | 0    |
| 2018-01-01 06:00:00 | 1    |
| 2018-01-01 12:00:00 | 2    |

</details>

inserting to postgres
```python
df_test.to_sql('foo_table',postgres_uri,schema='data_test',index_label='info_date')
```
when reading from the database I get the index with different type
```python
df_db = pd.read_sql_table('foo_table',postgres_uri,schema='data_test',index_col='info_date')
```
<details>

|                           | nums |
|---------------------------|------|
| info_date                 |      |
| 2018-01-01 00:00:00+00:00 | 0    |
| 2018-01-01 06:00:00+00:00 | 1    |
| 2018-01-01 12:00:00+00:00 | 2    |

</details>
As you can see `info_date` is of type `datetime64[ns, UTC]`

not sure if it's a sqlalchemy or pandas question.
reported because of my [question](https://stackoverflow.com/questions/52986158/pandas-to-sql-turns-datetime64-index-to-time-zone-aware) on SO


Discussion:

Resolving patch:
diff --git a/pandas/core/generic.py b/pandas/core/generic.py
--- a/pandas/core/generic.py
+++ b/pandas/core/generic.py
@@ -2397,6 +2397,15 @@ def to_sql(self, name, con, schema=None, if_exists='fail', index=True,
         --------
         pandas.read_sql : read a DataFrame from a table
 
+        Notes
+        -----
+        Timezone aware datetime columns will be written as
+        ``Timestamp with timezone`` type with SQLAlchemy if supported by the
+        database. Otherwise, the datetimes will be stored as timezone unaware
+        timestamps local to the original timezone.
+
+        .. versionadded:: 0.24.0
+
         References
         ----------
         .. [1] http://docs.sqlalchemy.org
diff --git a/pandas/io/sql.py b/pandas/io/sql.py
--- a/pandas/io/sql.py
+++ b/pandas/io/sql.py
@@ -592,12 +592,17 @@ def insert_data(self):
         data_list = [None] * ncols
         blocks = temp._data.blocks
 
-        for i in range(len(blocks)):
-            b = blocks[i]
+        for b in blocks:
             if b.is_datetime:
-                # convert to microsecond resolution so this yields
-                # datetime.datetime
-                d = b.values.astype('M8[us]').astype(object)
+                # return datetime.datetime objects
+                if b.is_datetimetz:
+                    # GH 9086: Ensure we return datetimes with timezone info
+                    # Need to return 2-D data; DatetimeIndex is 1D
+                    d = b.values.to_pydatetime()
+                    d = np.expand_dims(d, axis=0)
+                else:
+                    # convert to microsecond resolution for datetime.datetime
+                    d = b.values.astype('M8[us]').astype(object)
             else:
                 d = np.array(b.get_values(), dtype=object)
 
@@ -612,7 +617,7 @@ def insert_data(self):
         return column_names, data_list
 
     def _execute_insert(self, conn, keys, data_iter):
-        data = [{k: v for k, v in zip(keys, row)} for row in data_iter]
+        data = [dict(zip(keys, row)) for row in data_iter]
         conn.execute(self.insert_statement(), data)
 
     def insert(self, chunksize=None):
@@ -741,8 +746,9 @@ def _get_column_names_and_types(self, dtype_mapper):
     def _create_table_setup(self):
         from sqlalchemy import Table, Column, PrimaryKeyConstraint
 
-        column_names_and_types = \
-            self._get_column_names_and_types(self._sqlalchemy_type)
+        column_names_and_types = self._get_column_names_and_types(
+            self._sqlalchemy_type
+        )
 
         columns = [Column(name, typ, index=is_index)
                    for name, typ, is_index in column_names_and_types]
@@ -841,14 +847,19 @@ def _sqlalchemy_type(self, col):
 
         from sqlalchemy.types import (BigInteger, Integer, Float,
                                       Text, Boolean,
-                                      DateTime, Date, Time)
+                                      DateTime, Date, Time, TIMESTAMP)
 
         if col_type == 'datetime64' or col_type == 'datetime':
+            # GH 9086: TIMESTAMP is the suggested type if the column contains
+            # timezone information
             try:
-                tz = col.tzinfo  # noqa
-                return DateTime(timezone=True)
+                if col.dt.tz is not None:
+                    return TIMESTAMP(timezone=True)
             except AttributeError:
-                return DateTime
+                # The column is actually a DatetimeIndex
+                if col.tz is not None:
+                    return TIMESTAMP(timezone=True)
+            return DateTime
         if col_type == 'timedelta64':
             warnings.warn("the 'timedelta' type is not supported, and will be "
                           "written as integer values (ns frequency) to the "
@@ -1275,8 +1286,9 @@ def _create_table_setup(self):
         structure of a DataFrame.  The first entry will be a CREATE TABLE
         statement while the rest will be CREATE INDEX statements.
         """
-        column_names_and_types = \
-            self._get_column_names_and_types(self._sql_type_name)
+        column_names_and_types = self._get_column_names_and_types(
+            self._sql_type_name
+        )
 
         pat = re.compile(r'\s+')
         column_names = [col_name for col_name, _, _ in column_names_and_types]


Test patch:
diff --git a/pandas/tests/io/test_sql.py b/pandas/tests/io/test_sql.py
--- a/pandas/tests/io/test_sql.py
+++ b/pandas/tests/io/test_sql.py
@@ -961,7 +961,8 @@ def test_sqlalchemy_type_mapping(self):
                                             utc=True)})
         db = sql.SQLDatabase(self.conn)
         table = sql.SQLTable("test_type", db, frame=df)
-        assert isinstance(table.table.c['time'].type, sqltypes.DateTime)
+        # GH 9086: TIMESTAMP is the suggested type for datetimes with timezones
+        assert isinstance(table.table.c['time'].type, sqltypes.TIMESTAMP)
 
     def test_database_uri_string(self):
 
@@ -1361,9 +1362,51 @@ def check(col):
         df = sql.read_sql_table("types_test_data", self.conn)
         check(df.DateColWithTz)
 
+    def test_datetime_with_timezone_roundtrip(self):
+        # GH 9086
+        # Write datetimetz data to a db and read it back
+        # For dbs that support timestamps with timezones, should get back UTC
+        # otherwise naive data should be returned
+        expected = DataFrame({'A': date_range(
+            '2013-01-01 09:00:00', periods=3, tz='US/Pacific'
+        )})
+        expected.to_sql('test_datetime_tz', self.conn, index=False)
+
+        if self.flavor == 'postgresql':
+            # SQLAlchemy "timezones" (i.e. offsets) are coerced to UTC
+            expected['A'] = expected['A'].dt.tz_convert('UTC')
+        else:
+            # Otherwise, timestamps are returned as local, naive
+            expected['A'] = expected['A'].dt.tz_localize(None)
+
+        result = sql.read_sql_table('test_datetime_tz', self.conn)
+        tm.assert_frame_equal(result, expected)
+
+        result = sql.read_sql_query(
+            'SELECT * FROM test_datetime_tz', self.conn
+        )
+        if self.flavor == 'sqlite':
+            # read_sql_query does not return datetime type like read_sql_table
+            assert isinstance(result.loc[0, 'A'], string_types)
+            result['A'] = to_datetime(result['A'])
+        tm.assert_frame_equal(result, expected)
+
+    def test_naive_datetimeindex_roundtrip(self):
+        # GH 23510
+        # Ensure that a naive DatetimeIndex isn't converted to UTC
+        dates = date_range('2018-01-01', periods=5, freq='6H')
+        expected = DataFrame({'nums': range(5)}, index=dates)
+        expected.to_sql('foo_table', self.conn, index_label='info_date')
+        result = sql.read_sql_table('foo_table', self.conn,
+                                    index_col='info_date')
+        # result index with gain a name from a set_index operation; expected
+        tm.assert_frame_equal(result, expected, check_names=False)
+
     def test_date_parsing(self):
         # No Parsing
         df = sql.read_sql_table("types_test_data", self.conn)
+        expected_type = object if self.flavor == 'sqlite' else np.datetime64
+        assert issubclass(df.DateCol.dtype.type, expected_type)
 
         df = sql.read_sql_table("types_test_data", self.conn,
                                 parse_dates=['DateCol'])