Repository: frictionlessdata/frictionless-py

Issue:
Exception on pandas parser when using a datetime column containing timezone information as index 
If an user try to load a resource and use a datetime column as index, if this column has timezone information the transform to pandas (.to_pandas()) will give an exception:

`ValueError: Tz-aware datetime.datetime cannot be converted to datetime64 unless utc=True`

An example to reproduce this problem:

datapackage.json

```

{
    "name": "to_pandas_bug",
    "resources": [
        {
            "profile": "tabular-data-resource",
            "name": "timezone-issue",
            "path": "timezone-issue.csv",
            "scheme": "file",
            "format": "csv",
            "encoding": "utf-8",
            "schema": {
                "fields": [
                    {"name": "id", "type": "integer"},
                    {
                        "name": "timestamp",
                        "type": "datetime"
                    }
                ],
                "primaryKey": ["timestamp"]
            },
            "dialect": {"delimiter": ";"}
        }
    ]
}
```
And the data timezone-issue.csv:

```
id;timestamp
0;2020-01-01T15:00:00+03:00
1;2020-01-01T15:00:00+04:00
```
@roll 

Discussion:

Patch:
diff --git a/data/issue-1105-timezone.csv b/data/issue-1105-timezone.csv
new file mode 100644
index 00000000..cc6f198b
--- /dev/null
+++ b/data/issue-1105-timezone.csv
@@ -0,0 +1,3 @@
+id;timestamp
+0;2020-01-01T15:00:00Z
+1;2020-01-01T15:00:00+03:00
diff --git a/data/issue-1105.package.json b/data/issue-1105.package.json
new file mode 100644
index 00000000..6f401ba8
--- /dev/null
+++ b/data/issue-1105.package.json
@@ -0,0 +1,24 @@
+{
+    "name": "to_pandas_bug",
+    "resources": [
+        {
+            "profile": "tabular-data-resource",
+            "name": "issue1105",
+            "path": "issue-1105-timezone.csv",
+            "scheme": "file",
+            "format": "csv",
+            "encoding": "utf-8",
+            "schema": {
+                "fields": [
+                    {"name": "id", "type": "integer"},
+                    {
+                        "name": "timestamp",
+                        "type": "datetime"
+                    }
+                ],
+                "primaryKey": ["timestamp"]
+            },
+            "dialect": {"delimiter": ";"}
+        }
+    ]
+}
diff --git a/frictionless/plugins/pandas/parser.py b/frictionless/plugins/pandas/parser.py
index a4ed9213..23fcf4a8 100644
--- a/frictionless/plugins/pandas/parser.py
+++ b/frictionless/plugins/pandas/parser.py
@@ -141,6 +141,8 @@ class PandasParser(Parser):
                 data_rows.append(tuple(data_values))
 
         # Create index
+        pd = helpers.import_from_plugin("pandas", plugin="pandas")
+
         index = None
         if source.schema.primary_key:
             if len(source.schema.primary_key) == 1:
@@ -149,7 +151,9 @@ class PandasParser(Parser):
                 index_dtype = self.__write_convert_type(index_field.type)
                 if index_field.type in ["datetime", "date"]:
                     index_class = pd.DatetimeIndex
+                    index_rows = pd.to_datetime(index_rows, utc=True)
                 index = index_class(index_rows, name=index_field.name, dtype=index_dtype)
+
             elif len(source.schema.primary_key) > 1:
                 index = pd.MultiIndex.from_tuples(
                     index_rows, names=source.schema.primary_key
@@ -167,12 +171,13 @@ class PandasParser(Parser):
 
     def __write_convert_type(self, type=None):
         np = helpers.import_from_plugin("numpy", plugin="pandas")
+        pd = helpers.import_from_plugin("pandas", plugin="pandas")
 
         # Mapping
         mapping = {
             "array": np.dtype(list),
             "boolean": np.dtype(bool),
-            "datetime": np.dtype("datetime64[ns]"),
+            "datetime": pd.DatetimeTZDtype(tz="UTC"),
             "integer": np.dtype(int),
             "number": np.dtype(float),
             "object": np.dtype(dict),


Test patch:
diff --git a/tests/plugins/pandas/test_parser.py b/tests/plugins/pandas/test_parser.py
index 84a4808a..7c2342dc 100644
--- a/tests/plugins/pandas/test_parser.py
+++ b/tests/plugins/pandas/test_parser.py
@@ -42,6 +42,17 @@ def test_pandas_parser_write_bug_1100():
     ]
 
 
+def test_pandas_parser_write_bug_1105():
+    datapackage = Package("data/issue-1105.package.json")
+    target = datapackage.resources[0].to_pandas()
+    assert target.to_dict() == {
+        "id": {
+            pd.Timestamp("2020-01-01 12:00:00+0000", tz="UTC"): 1,
+            pd.Timestamp("2020-01-01 15:00:00+0000", tz="UTC"): 0,
+        }
+    }
+
+
 def test_pandas_parser_write_types():
     source = Package("data/storage/types.json").get_resource("types")
     target = source.write(format="pandas")