Repository: frictionlessdata/frictionless-py

Issue:
Resource function to_pandas fails creating index if last field is of type 'datetime'
When reading a resource using `to_pandas()`, index is build from primary key.
But while creating index, code looks into last looked-up field instead of index field to get index data type.
If last field type is of type `datetime` or `date`, index column is tried to be converted to DatetimeIndex, but this fails if primary key column is of type `integer`.

I will provide example package.json and propose a fix in PR.

Discussion:

Patch:
diff --git a/data/fixtures/cli/long-error-messages-976.txt b/data/fixtures/cli/long-error-messages-976.txt
new file mode 100644
index 00000000..5617a77e
--- /dev/null
+++ b/data/fixtures/cli/long-error-messages-976.txt
@@ -0,0 +1,20 @@
+# -------
+# invalid: test-tabulator 
+# -------
+
+row    field    code       message
+-----  -------  ---------  -------------------------------------------------
+                resource-  The data resource has an error: "{'format':
+                error      'inline', 'hashing': 'md5', 'name': 'test-
+                           tabulator', 'profile': 'tabular-data-resource',
+                           'resources': [{'name': 'first-resource', 'path':
+                           'table.xls', 'schema': {'fields': [{'name': 'id',
+                           'type': 'number'}, {'name': 'name', 'type':
+                           'string'}]}}, {'name': 'number-two', 'path':
+                           'table-reverse.csv', 'schema': {'fields':
+                           [{'name': 'id', 'type': 'integer'}, {'name':
+                           'name', 'type': 'string'}]}}], 'scheme': '',
+                           'stats': {'bytes': 0, 'fields': 0, 'hash': '',
+                           'rows': 0}} is not valid under any of the given
+                           schemas" at "" in metadata and at "oneOf" in
+                           profile
\ No newline at end of file
diff --git a/data/fixtures/cli/zipped-resources-979.txt b/data/fixtures/cli/zipped-resources-979.txt
new file mode 100644
index 00000000..9cd498c4
--- /dev/null
+++ b/data/fixtures/cli/zipped-resources-979.txt
@@ -0,0 +1,15 @@
+# -----
+# valid: ogd10_energieforschungstatistik_ch.csv 
+# -----
+# -------
+# invalid: ogd10_catalogs.zip => capital-invalid.csv 
+# -------
+
+row    field    code     message
+-----  -------  -------  -------------------------------------------------
+                schema-  Schema is not valid: Schemas with duplicate field
+                error    names are not supported
+
+# -----
+# valid: ogd10_catalogs.zip => finanzquellen.csv 
+# -----
\ No newline at end of file
diff --git a/data/issue-1100.csv b/data/issue-1100.csv
new file mode 100644
index 00000000..90f6a4d1
--- /dev/null
+++ b/data/issue-1100.csv
@@ -0,0 +1,3 @@
+id;timestamp
+0;2022-05-25 10.39.15
+1;2022-05-25 10.39.15
diff --git a/data/issue-1100.package.json b/data/issue-1100.package.json
new file mode 100644
index 00000000..65434b9e
--- /dev/null
+++ b/data/issue-1100.package.json
@@ -0,0 +1,25 @@
+{
+    "name": "to_pandas_bug",
+    "resources": [
+        {
+            "profile": "tabular-data-resource",
+            "name": "issue1100",
+            "path": "issue-1100.csv",
+            "scheme": "file",
+            "format": "csv",
+            "encoding": "utf-8",
+            "schema": {
+                "fields": [
+                    {"name": "id", "type": "integer"},
+                    {
+                        "name": "timestamp",
+                        "type": "datetime",
+                        "format": "%Y-%m-%d %H.%M.%S"
+                    }
+                ],
+                "primaryKey": ["id"]
+            },
+            "dialect": {"delimiter": ";"}
+        }
+    ]
+}
diff --git a/frictionless/plugins/pandas/parser.py b/frictionless/plugins/pandas/parser.py
index 3aeedc4c..a4ed9213 100644
--- a/frictionless/plugins/pandas/parser.py
+++ b/frictionless/plugins/pandas/parser.py
@@ -1,5 +1,6 @@
 import isodate
 import datetime
+import decimal
 from ...parser import Parser
 from ...schema import Schema
 from ...field import Field
@@ -42,8 +43,6 @@ class PandasParser(Parser):
                     value = item[field.name]
                 if field.type == "number" and np.isnan(value):
                     value = None
-                elif field.type == "datetime":
-                    value = value.to_pydatetime()
                 cells.append(value)
             yield cells
 
@@ -88,6 +87,8 @@ class PandasParser(Parser):
         if sample is not None:
             if isinstance(sample, (list, tuple)):
                 return "array"
+            elif isinstance(sample, datetime.datetime):
+                return "datetime"
             elif isinstance(sample, datetime.date):
                 return "date"
             elif isinstance(sample, isodate.Duration):
@@ -122,16 +123,17 @@ class PandasParser(Parser):
                     value = row[field.name]
                     if isinstance(value, float) and np.isnan(value):
                         value = None
+                    if isinstance(value, decimal.Decimal):
+                        value = float(value)
                     # http://pandas.pydata.org/pandas-docs/stable/gotchas.html#support-for-integer-na
                     if value is None and field.type in ("number", "integer"):
                         fixed_types[field.name] = "number"
                         value = np.NaN
-                    if field.type in ["datetime", "time"] and value is not None:
-                        value = value.replace(tzinfo=None)
                     if field.name in source.schema.primary_key:
                         index_values.append(value)
                     else:
                         data_values.append(value)
+
                 if len(source.schema.primary_key) == 1:
                     index_rows.append(index_values[0])
                 elif len(source.schema.primary_key) > 1:
@@ -145,7 +147,7 @@ class PandasParser(Parser):
                 index_class = pd.Index
                 index_field = source.schema.get_field(source.schema.primary_key[0])
                 index_dtype = self.__write_convert_type(index_field.type)
-                if field.type in ["datetime", "date"]:
+                if index_field.type in ["datetime", "date"]:
                     index_class = pd.DatetimeIndex
                 index = index_class(index_rows, name=index_field.name, dtype=index_dtype)
             elif len(source.schema.primary_key) > 1:
@@ -154,17 +156,13 @@ class PandasParser(Parser):
                 )
 
         # Create dtypes/columns
-        dtypes = []
         columns = []
         for field in source.schema.fields:
             if field.name not in source.schema.primary_key:
-                dtype = self.__write_convert_type(fixed_types.get(field.name, field.type))
-                dtypes.append((field.name, dtype))
                 columns.append(field.name)
 
         # Create/set dataframe
-        array = np.array(data_rows, dtype=dtypes)
-        dataframe = pd.DataFrame(array, index=index, columns=columns)
+        dataframe = pd.DataFrame(data_rows, index=index, columns=columns)
         target.data = dataframe
 
     def __write_convert_type(self, type=None):
diff --git a/frictionless/program/validate.py b/frictionless/program/validate.py
index 54477a72..06004fb9 100644
--- a/frictionless/program/validate.py
+++ b/frictionless/program/validate.py
@@ -1,7 +1,9 @@
 import sys
 import petl
 import typer
+import textwrap
 from typing import List
+from tabulate import tabulate
 from ..actions import validate
 from ..detector import Detector
 from ..dialect import Dialect
@@ -261,15 +263,41 @@ def program_validate(
                         error.message,
                     ]
                 )
+            content = _wrap_text_to_colwidths(content)
             typer.secho(
                 str(
-                    petl.util.vis.lookall(
-                        [["row", "field", "code", "message"]] + content,
-                        vrepr=str,
-                        style="simple",
+                    tabulate(
+                        content,
+                        headers=["row", "field", "code", "message"],
+                        tablefmt="simple",
                     )
                 )
             )
 
     # Return retcode
     raise typer.Exit(code=int(not report.valid))
+
+
+# TODO:This is a temporary function to use with tabulate as
+# tabulate 0.8.9 does not support text wrap
+def _wrap_text_to_colwidths(
+    list_of_lists: List, colwidths: List = [5, 5, 10, 50]
+) -> List:
+    """Create new list with wrapped text with different column width.
+    Args:
+        list_of_lists (List): List of lines
+        colwidths (List): width for each column
+
+    Returns:
+        List: list of lines with wrapped text
+
+    """
+    result = []
+    for row in list_of_lists:
+        new_row = []
+        for cell, width in zip(row, colwidths):
+            cell = str(cell)
+            wrapped = textwrap.wrap(cell, width=width)
+            new_row.append("\n".join(wrapped))
+        result.append(new_row)
+    return result


Test patch:
diff --git a/tests/plugins/bigquery/test_parser.py b/tests/plugins/bigquery/test_parser.py
index cbe485f7..480e045a 100644
--- a/tests/plugins/bigquery/test_parser.py
+++ b/tests/plugins/bigquery/test_parser.py
@@ -27,6 +27,7 @@ def test_bigquery_parser_write(options):
         ]
 
 
+# TODO: add timezone support or document if it's not possible
 @pytest.mark.ci
 def test_bigquery_parser_write_timezone(options):
     prefix = options.pop("prefix")
diff --git a/tests/plugins/ckan/test_parser.py b/tests/plugins/ckan/test_parser.py
index 9bb46e74..8019a1be 100644
--- a/tests/plugins/ckan/test_parser.py
+++ b/tests/plugins/ckan/test_parser.py
@@ -21,6 +21,7 @@ def test_ckan_parser(options):
         ]
 
 
+# TODO: add timezone support or document if it's not possible
 @pytest.mark.vcr
 def test_ckan_parser_timezone(options):
     url = options.pop("url")
diff --git a/tests/plugins/pandas/test_parser.py b/tests/plugins/pandas/test_parser.py
index 87f4960b..84a4808a 100644
--- a/tests/plugins/pandas/test_parser.py
+++ b/tests/plugins/pandas/test_parser.py
@@ -1,6 +1,8 @@
 import pytz
 import isodate
 import datetime
+from dateutil.tz import tzutc
+from dateutil.tz import tzoffset
 import pandas as pd
 from decimal import Decimal
 from frictionless import Package, Resource
@@ -31,6 +33,15 @@ def test_pandas_parser_write():
     ]
 
 
+def test_pandas_parser_write_bug_1100():
+    datapackage = Package("data/issue-1100.package.json")
+    target = datapackage.resources[0].to_pandas()
+    assert target.to_dict("records") == [
+        {"timestamp": pd.Timestamp(2022, 5, 25, 10, 39, 15)},
+        {"timestamp": pd.Timestamp(2022, 5, 25, 10, 39, 15)},
+    ]
+
+
 def test_pandas_parser_write_types():
     source = Package("data/storage/types.json").get_resource("types")
     target = source.write(format="pandas")
@@ -129,9 +140,22 @@ def test_pandas_parser_write_timezone():
         # Assert rows
         assert target.read_rows() == [
             {"datetime": datetime.datetime(2020, 1, 1, 15), "time": datetime.time(15)},
-            {"datetime": datetime.datetime(2020, 1, 1, 15), "time": datetime.time(15)},
-            {"datetime": datetime.datetime(2020, 1, 1, 15), "time": datetime.time(15)},
-            {"datetime": datetime.datetime(2020, 1, 1, 15), "time": datetime.time(15)},
+            {
+                "datetime": datetime.datetime(2020, 1, 1, 15, 0, tzinfo=tzutc()),
+                "time": datetime.time(15, 0, tzinfo=tzutc()),
+            },
+            {
+                "datetime": datetime.datetime(
+                    2020, 1, 1, 15, 0, tzinfo=tzoffset(None, 10800)
+                ),
+                "time": datetime.time(15, 0, tzinfo=tzoffset(None, 10800)),
+            },
+            {
+                "datetime": datetime.datetime(
+                    2020, 1, 1, 15, 0, tzinfo=tzoffset(None, -10800)
+                ),
+                "time": datetime.time(15, 0, tzinfo=tzoffset(None, -10800)),
+            },
         ]
 
 
diff --git a/tests/plugins/spss/test_parser.py b/tests/plugins/spss/test_parser.py
index d3539958..c5ebe197 100644
--- a/tests/plugins/spss/test_parser.py
+++ b/tests/plugins/spss/test_parser.py
@@ -109,6 +109,7 @@ def test_spss_storage_constraints(tmpdir):
             ]
 
 
+# TODO: add timezone support or document if it's not possible
 @pytest.mark.skipif(sys.version_info >= (3, 10), reason="Not supported Python3.10")
 def test_spss_parser_write_timezone(tmpdir):
     source = Resource("data/timezone.csv")
diff --git a/tests/plugins/sql/parser/test_mysql.py b/tests/plugins/sql/parser/test_mysql.py
index aafc6b18..2c9ac10c 100644
--- a/tests/plugins/sql/parser/test_mysql.py
+++ b/tests/plugins/sql/parser/test_mysql.py
@@ -6,6 +6,7 @@ from frictionless.plugins.sql import SqlDialect
 # General
 
 
+# TODO: add timezone support or document if it's not possible
 def test_sql_parser_write_timezone_mysql(mysql_url):
     source = Resource("data/timezone.csv")
     target = source.write(mysql_url, dialect=SqlDialect(table="timezone"))
diff --git a/tests/plugins/sql/parser/test_postgres.py b/tests/plugins/sql/parser/test_postgres.py
index 415eef25..7bcd6ca8 100644
--- a/tests/plugins/sql/parser/test_postgres.py
+++ b/tests/plugins/sql/parser/test_postgres.py
@@ -6,6 +6,7 @@ from frictionless.plugins.sql import SqlDialect
 # General
 
 
+# TODO: add timezone support or document if it's not possible
 def test_sql_parser_write_timezone_postgresql(postgresql_url):
     source = Resource("data/timezone.csv")
     target = source.write(postgresql_url, dialect=SqlDialect(table="timezone"))
diff --git a/tests/plugins/sql/parser/test_sqlite.py b/tests/plugins/sql/parser/test_sqlite.py
index a95539f5..d05ffb53 100644
--- a/tests/plugins/sql/parser/test_sqlite.py
+++ b/tests/plugins/sql/parser/test_sqlite.py
@@ -98,6 +98,7 @@ def test_sql_parser_write_where(database_url):
         ]
 
 
+# TODO: add timezone support or document if it's not possible
 def test_sql_parser_write_timezone(sqlite_url):
     source = Resource("data/timezone.csv")
     target = source.write(sqlite_url, dialect=SqlDialect(table="timezone"))
diff --git a/tests/program/test_validate.py b/tests/program/test_validate.py
index 751d3fb3..90804564 100644
--- a/tests/program/test_validate.py
+++ b/tests/program/test_validate.py
@@ -215,13 +215,24 @@ def test_program_validate_error_not_found():
 
 def test_program_validate_zipped_resources_979():
     result = runner.invoke(program, "validate data/zipped-resources/datapackage.json")
+    output_file_path = "data/fixtures/cli/zipped-resources-979.txt"
+    with open(output_file_path, encoding="utf-8") as file:
+        expected = file.read()
     assert result.exit_code == 1
     assert result.stdout.count("valid: ogd10_energieforschungstatistik_ch.csv")
     assert result.stdout.count("valid: ogd10_catalogs.zip => finanzquellen.csv")
     assert result.stdout.count("invalid: ogd10_catalogs.zip => capital-invalid.csv")
-    assert result.stdout.count(
-        "Schema is not valid: Schemas with duplicate field names are not supported"
-    )
+    assert result.stdout.count("Schema is not valid")
+    assert result.stdout.strip() == expected.strip()
+
+
+def test_program_validate_long_error_messages_976():
+    result = runner.invoke(program, "validate data/datapackage.json --type resource")
+    output_file_path = "data/fixtures/cli/long-error-messages-976.txt"
+    with open(output_file_path, encoding="utf-8") as file:
+        expected = file.read()
+    assert result.exit_code == 1
+    assert result.stdout.strip() == expected.strip()
 
 
 # Helpers