Repository: frictionlessdata/frictionless-py

Issue:
Metadata.to_yaml() fails with error "cannot represent an object"
**Overview**

Greetings!

It testing out the most recent version of `frictionless` (both pypi and Github master branch), `Metadata.to_yaml()` (as called through `Package.yaml()`) fails with an error message:

```
In [28]: package.to_yaml('test.yml')
---------------------------------------------------------------------------
RepresenterError                          Traceback (most recent call last)
~/.local/lib/python3.8/site-packages/frictionless/metadata.py in to_yaml(self, target)
    137             with tempfile.NamedTemporaryFile("wt", delete=False) as file:
--> 138                 yaml.dump(self.to_dict(), file, Dumper=IndentDumper)
    139             helpers.move_file(file.name, target)

/usr/lib/python3.8/site-packages/yaml/__init__.py in dump(data, stream, Dumper, **kwds)
    289     """
--> 290     return dump_all([data], stream, Dumper=Dumper, **kwds)
    291 

/usr/lib/python3.8/site-packages/yaml/__init__.py in dump_all(documents, stream, Dumper, default_style, default_flow_style, canonical, indent, width, allow_unicode, line_break, encoding, explicit_start, explicit_end, version, tags, sort_keys)
    277         for data in documents:
--> 278             dumper.represent(data)
    279         dumper.close()

/usr/lib/python3.8/site-packages/yaml/representer.py in represent(self, data)
     26     def represent(self, data):
---> 27         node = self.represent_data(data)
     28         self.serialize(node)

/usr/lib/python3.8/site-packages/yaml/representer.py in represent_data(self, data)
     47         if data_types[0] in self.yaml_representers:
---> 48             node = self.yaml_representers[data_types[0]](self, data)
     49         else:

/usr/lib/python3.8/site-packages/yaml/representer.py in represent_dict(self, data)
    206     def represent_dict(self, data):
--> 207         return self.represent_mapping('tag:yaml.org,2002:map', data)
    208 

/usr/lib/python3.8/site-packages/yaml/representer.py in represent_mapping(self, tag, mapping, flow_style)
    117             node_key = self.represent_data(item_key)
--> 118             node_value = self.represent_data(item_value)
    119             if not (isinstance(node_key, ScalarNode) and not node_key.style):

/usr/lib/python3.8/site-packages/yaml/representer.py in represent_data(self, data)
     57                 elif None in self.yaml_representers:
---> 58                     node = self.yaml_representers[None](self, data)
     59                 else:

/usr/lib/python3.8/site-packages/yaml/representer.py in represent_undefined(self, data)
    230     def represent_undefined(self, data):
--> 231         raise RepresenterError("cannot represent an object", data)
    232 

RepresenterError: ('cannot represent an object', [{'path': 'test/data/data.csv', 'control': {'newline': ''}, 'schema': {'fields': [{'name': 'atomic number', 'type': 'integer'}, {'name': 'symbol', 'type': 'string'}, {'name': 'name', 'type': 'string'}, {'name': 'atomic mass', 'type': 'number'}, {'name': 'metal or nonmetal?', 'type': 'string'}]}, 'profile': 'tabular-data-resource', 'name': 'data', 'scheme': 'file', 'format': 'csv', 'hashing': 'md5', 'encoding': 'utf-8', 'compression': 'no', 'compressionPath': '', 'dialect': {}, 'query': {}, 'stats': {'hash': 'ec86e3663d1e497ff100dc66002cb88b', 'bytes': 4252, 'fields': 5, 'rows': 118}}])
```

The issue occurs with using both the command-line tool and the library itself.

It seems like pyyaml may be unhappy about encountering `frictionless.helpers.ControlledList` objects in the dict representation of the package created during the call to `Package.to_yaml()`?

To reproduce the issue, one can simply follow the steps from the `Describing Packages` section of [the frictionless tutorial](https://github.com/frictionlessdata/frictionless-py/blob/master/docs/target/describing-data/README.md).

Attempting to run:

```
frictionless describe *-3.csv
```

Should produce a similar issue to the above.


**System Info**

- Arch Linux 5.8.12 64-bit
- Python 3.8.5
- frictionless 3.15.0 (commit 7c45cfd)
- py-yaml 5.3.1

---

Please preserve this line to notify @roll (lead of this repository)


Discussion:

Patch:
diff --git a/frictionless/controls.py b/frictionless/controls.py
index 0d770508..54678cb7 100644
--- a/frictionless/controls.py
+++ b/frictionless/controls.py
@@ -48,16 +48,6 @@ class Control(Metadata):
     def expand(self):
         pass
 
-    # Import/Export
-
-    def to_dict(self, expand=False):
-        result = super().to_dict()
-        if expand:
-            result = type(self)(result)
-            result.expand()
-            result = result.to_dict()
-        return result
-
     # Metadata
 
     metadata_Error = errors.ControlError
diff --git a/frictionless/dialects.py b/frictionless/dialects.py
index 352090fc..1377ccc8 100644
--- a/frictionless/dialects.py
+++ b/frictionless/dialects.py
@@ -60,21 +60,6 @@ class Dialect(Metadata):
         self.setdefault("headerRows", self.header_rows)
         self.setdefault("headerJoin", self.header_join)
 
-    # Import/Export
-
-    def to_dict(self, expand=False):
-        """Conver to a dict
-
-        Parameters:
-            expand (bool): if True call `metadata.expand` for the exported copy
-        """
-        result = super().to_dict()
-        if expand:
-            result = type(self)(result)
-            result.expand()
-            result = result.to_dict()
-        return result
-
     # Metadata
 
     metadata_Error = errors.DialectError
diff --git a/frictionless/field.py b/frictionless/field.py
index 73c66ac5..10ce829d 100644
--- a/frictionless/field.py
+++ b/frictionless/field.py
@@ -295,21 +295,6 @@ class Field(Metadata):
         """
         return self.__type.write_cell(cell)
 
-    # Import/Export
-
-    def to_dict(self, expand=False):
-        """Convert field to dict
-
-        Parameters:
-            expand (bool): whether to expand
-        """
-        result = super().to_dict()
-        if expand:
-            result = type(self)(result)
-            result.expand()
-            result = result.to_dict()
-        return result
-
     # Metadata
 
     def metadata_process(self):
diff --git a/frictionless/helpers.py b/frictionless/helpers.py
index 3a379684..18206962 100644
--- a/frictionless/helpers.py
+++ b/frictionless/helpers.py
@@ -245,33 +245,6 @@ def detect_source_scheme_and_format(source):
     return (scheme, format)
 
 
-def get_current_memory_usage():
-    # Current memory usage of the current process in MB
-    # This will only work on systems with a /proc file system (like Linux)
-    # https://stackoverflow.com/questions/897941/python-equivalent-of-phps-memory-get-usage
-    try:
-        with open("/proc/self/status") as status:
-            for line in status:
-                parts = line.split()
-                key = parts[0][2:-1].lower()
-                if key == "rss":
-                    return int(parts[1]) / 1000
-    except Exception:
-        pass
-
-
-class Timer:
-    def __init__(self):
-        self.__start = datetime.datetime.now()
-        self.__stop = None
-
-    @property
-    def time(self):
-        if not self.__stop:
-            self.__stop = datetime.datetime.now()
-        return round((self.__stop - self.__start).total_seconds(), 3)
-
-
 # Collections
 
 
@@ -370,6 +343,44 @@ class ControlledList(list):
         return result
 
 
+def deepnative(value):
+    if isinstance(value, dict):
+        value = {key: deepnative(value) for key, value in value.items()}
+    elif isinstance(value, list):
+        value = [deepnative(value) for value in value]
+    return value
+
+
+# Measurements
+
+
+class Timer:
+    def __init__(self):
+        self.__start = datetime.datetime.now()
+        self.__stop = None
+
+    @property
+    def time(self):
+        if not self.__stop:
+            self.__stop = datetime.datetime.now()
+        return round((self.__stop - self.__start).total_seconds(), 3)
+
+
+def get_current_memory_usage():
+    # Current memory usage of the current process in MB
+    # This will only work on systems with a /proc file system (like Linux)
+    # https://stackoverflow.com/questions/897941/python-equivalent-of-phps-memory-get-usage
+    try:
+        with open("/proc/self/status") as status:
+            for line in status:
+                parts = line.split()
+                key = parts[0][2:-1].lower()
+                if key == "rss":
+                    return int(parts[1]) / 1000
+    except Exception:
+        pass
+
+
 # Backports
 
 
diff --git a/frictionless/metadata.py b/frictionless/metadata.py
index 948531bd..7d49bd26 100644
--- a/frictionless/metadata.py
+++ b/frictionless/metadata.py
@@ -96,7 +96,7 @@ class Metadata(helpers.ControlledDict):
         Returns:
             dict: metadata as a dict
         """
-        return self.copy()
+        return helpers.deepnative(self)
 
     # NOTE: improve this code
     def to_json(self, target=None, encoder_class=None):
diff --git a/frictionless/package.py b/frictionless/package.py
index 2d11734c..96ac165f 100644
--- a/frictionless/package.py
+++ b/frictionless/package.py
@@ -324,6 +324,37 @@ class Package(Metadata):
         descriptor = {key: value for key, value in self.items() if key != "resources"}
         return Package(descriptor, resources=resources)
 
+    # NOTE: support multipart
+    def to_zip(self, target, encoder_class=None):
+        """Save package to a zip
+
+        Parameters:
+            target (str): target path
+
+        Raises:
+            FrictionlessException: on any error
+        """
+        try:
+            with zipfile.ZipFile(target, "w") as zip:
+                descriptor = self.copy()
+                for resource in self.resources:
+                    if resource.inline:
+                        continue
+                    if resource.remote:
+                        continue
+                    if resource.multipart:
+                        continue
+                    if not helpers.is_safe_path(resource.path):
+                        continue
+                    zip.write(resource.source, resource.path)
+                descriptor = json.dumps(
+                    descriptor, indent=2, ensure_ascii=False, cls=encoder_class
+                )
+                zip.writestr("datapackage.json", descriptor)
+        except Exception as exception:
+            error = errors.PackageError(note=str(exception))
+            raise exceptions.FrictionlessException(error) from exception
+
     def to_storage(self, storage, *, force=False):
         """Export package to storage
 
@@ -386,53 +417,6 @@ class Package(Metadata):
             force=force,
         )
 
-    def to_dict(self, expand=False):
-        """Convert package to a dict
-
-        Parameters:
-            expand? (bool): return expanded metadata
-
-        Returns:
-            dict: package as a dict
-        """
-        result = super().to_dict()
-        if expand:
-            result = type(self)(result)
-            result.expand()
-            result = result.to_dict()
-        return result
-
-    # NOTE: support multipart
-    def to_zip(self, target, encoder_class=None):
-        """Save package to a zip
-
-        Parameters:
-            target (str): target path
-
-        Raises:
-            FrictionlessException: on any error
-        """
-        try:
-            with zipfile.ZipFile(target, "w") as zip:
-                descriptor = self.copy()
-                for resource in self.resources:
-                    if resource.inline:
-                        continue
-                    if resource.remote:
-                        continue
-                    if resource.multipart:
-                        continue
-                    if not helpers.is_safe_path(resource.path):
-                        continue
-                    zip.write(resource.source, resource.path)
-                descriptor = json.dumps(
-                    descriptor, indent=2, ensure_ascii=False, cls=encoder_class
-                )
-                zip.writestr("datapackage.json", descriptor)
-        except Exception as exception:
-            error = errors.PackageError(note=str(exception))
-            raise exceptions.FrictionlessException(error) from exception
-
     # Metadata
 
     metadata_duplicate = True
diff --git a/frictionless/query.py b/frictionless/query.py
index 5bdbbc54..3315f852 100644
--- a/frictionless/query.py
+++ b/frictionless/query.py
@@ -161,21 +161,6 @@ class Query(Metadata):
         """Expand metadata"""
         pass
 
-    # Import/Export
-
-    def to_dict(self, expand=False):
-        """Convert query to dict
-
-        Parameters:
-            expand (bool): whether to expand
-        """
-        result = super().to_dict()
-        if expand:
-            result = type(self)(result)
-            result.expand()
-            result = result.to_dict()
-        return result
-
     # Metadata
 
     metadata_Error = errors.QueryError
diff --git a/frictionless/report.py b/frictionless/report.py
index e3cc4cb1..d7ff7ab4 100644
--- a/frictionless/report.py
+++ b/frictionless/report.py
@@ -155,19 +155,6 @@ class Report(Metadata):
 
         return wrapper
 
-    def to_dict(self, expand=False):
-        """Convert field to dict
-
-        Parameters:
-            expand (bool): whether to expand
-        """
-        result = super().to_dict()
-        if expand:
-            result = type(self)(result)
-            result.expand()
-            result = result.to_dict()
-        return result
-
     # Metadata
 
     metadata_strict = True
@@ -406,21 +393,6 @@ class ReportTable(Metadata):
             result.append([context.get(prop) for prop in spec])
         return result
 
-    # Import/Export
-
-    def to_dict(self, expand=False):
-        """Convert field to dict
-
-        Parameters:
-            expand (bool): whether to expand
-        """
-        result = super().to_dict()
-        if expand:
-            result = type(self)(result)
-            result.expand()
-            result = result.to_dict()
-        return result
-
     # Metadata
 
     metadata_strict = True
diff --git a/frictionless/resource.py b/frictionless/resource.py
index 7050883a..ae30e168 100644
--- a/frictionless/resource.py
+++ b/frictionless/resource.py
@@ -673,73 +673,6 @@ class Resource(Metadata):
             name=name,
         )
 
-    def to_storage(self, storage, *, force=False):
-        """Export resource to storage
-
-        Parameters:
-            storage (Storage): storage instance
-            force (bool): overwrite existent
-        """
-        storage.write_resource(self.to_copy(), force=force)
-        return storage
-
-    def to_sql(self, *, engine, prefix="", namespace=None, force=False):
-        """Export resource to SQL table
-
-        Parameters:
-            engine (object): `sqlalchemy` engine
-            prefix (str): prefix for all tables
-            namespace (str): SQL scheme
-            force (bool): overwrite existent
-        """
-        return self.to_storage(
-            system.create_storage(
-                "sql", engine=engine, prefix=prefix, namespace=namespace
-            ),
-            force=force,
-        )
-
-    def to_pandas(self):
-        """Export resource to Pandas dataframe
-
-        Parameters:
-            dataframes (dict): pandas dataframes
-            force (bool): overwrite existent
-        """
-        return self.to_storage(system.create_storage("pandas"))
-
-    def to_spss(self, *, basepath, force=False):
-        """Export resource to SPSS file
-
-        Parameters:
-            basepath (str): SPSS dir path
-            force (bool): overwrite existent
-        """
-        return self.to_storage(
-            system.create_storage("spss", basepath=basepath), force=force
-        )
-
-    def to_bigquery(self, *, service, project, dataset, prefix="", force=False):
-        """Export resource to Bigquery table
-
-        Parameters:
-            service (object): BigQuery `Service` object
-            project (str): BigQuery project name
-            dataset (str): BigQuery dataset name
-            prefix? (str): prefix for all names
-            force (bool): overwrite existent
-        """
-        return self.to_storage(
-            system.create_storage(
-                "bigquery",
-                service=service,
-                project=project,
-                dataset=dataset,
-                prefix=prefix,
-            ),
-            force=force,
-        )
-
     def to_copy(self):
         """Create a copy of the resource"""
         if self.data and not isinstance(self.data, list):
@@ -748,19 +681,6 @@ class Resource(Metadata):
             return Resource(descriptor, data=self.data)
         return Resource(self)
 
-    def to_dict(self, expand=False):
-        """Convert resource to dict
-
-        Parameters:
-            expand (bool): whether to expand
-        """
-        result = super().to_dict()
-        if expand:
-            result = type(self)(result)
-            result.expand()
-            result = result.to_dict()
-        return result
-
     # NOTE: cache lookup?
     def to_table(self, **options):
         """Convert resource to Table
@@ -837,6 +757,73 @@ class Resource(Metadata):
             error = errors.ResourceError(note=str(exception))
             raise exceptions.FrictionlessException(error) from exception
 
+    def to_storage(self, storage, *, force=False):
+        """Export resource to storage
+
+        Parameters:
+            storage (Storage): storage instance
+            force (bool): overwrite existent
+        """
+        storage.write_resource(self.to_copy(), force=force)
+        return storage
+
+    def to_sql(self, *, engine, prefix="", namespace=None, force=False):
+        """Export resource to SQL table
+
+        Parameters:
+            engine (object): `sqlalchemy` engine
+            prefix (str): prefix for all tables
+            namespace (str): SQL scheme
+            force (bool): overwrite existent
+        """
+        return self.to_storage(
+            system.create_storage(
+                "sql", engine=engine, prefix=prefix, namespace=namespace
+            ),
+            force=force,
+        )
+
+    def to_pandas(self):
+        """Export resource to Pandas dataframe
+
+        Parameters:
+            dataframes (dict): pandas dataframes
+            force (bool): overwrite existent
+        """
+        return self.to_storage(system.create_storage("pandas"))
+
+    def to_spss(self, *, basepath, force=False):
+        """Export resource to SPSS file
+
+        Parameters:
+            basepath (str): SPSS dir path
+            force (bool): overwrite existent
+        """
+        return self.to_storage(
+            system.create_storage("spss", basepath=basepath), force=force
+        )
+
+    def to_bigquery(self, *, service, project, dataset, prefix="", force=False):
+        """Export resource to Bigquery table
+
+        Parameters:
+            service (object): BigQuery `Service` object
+            project (str): BigQuery project name
+            dataset (str): BigQuery dataset name
+            prefix? (str): prefix for all names
+            force (bool): overwrite existent
+        """
+        return self.to_storage(
+            system.create_storage(
+                "bigquery",
+                service=service,
+                project=project,
+                dataset=dataset,
+                prefix=prefix,
+            ),
+            force=force,
+        )
+
     # Metadata
 
     metadata_duplicate = True
diff --git a/frictionless/schema.py b/frictionless/schema.py
index 3be1ebb5..14121e3f 100644
--- a/frictionless/schema.py
+++ b/frictionless/schema.py
@@ -325,19 +325,6 @@ class Schema(Metadata):
         )
         return schema
 
-    def to_dict(self, expand=False):
-        """Convert resource to dict
-
-        Parameters:
-            expand (bool): whether to expand
-        """
-        result = super().to_dict()
-        if expand:
-            result = type(self)(result)
-            result.expand()
-            result = result.to_dict()
-        return result
-
     # Metadata
 
     metadata_duplicate = True


Test patch:
diff --git a/tests/test_package.py b/tests/test_package.py
index e634437e..f201615d 100644
--- a/tests/test_package.py
+++ b/tests/test_package.py
@@ -1,5 +1,6 @@
 import os
 import json
+import yaml
 import zipfile
 import pytest
 from frictionless import Package, Resource, exceptions
@@ -402,7 +403,7 @@ def test_package_infer_empty_file():
 def test_package_to_json(tmpdir):
 
     # Write
-    target = os.path.join(tmpdir, "datapackage.json")
+    target = os.path.join(tmpdir, "package.json")
     package = Package("data/package.json")
     package.to_json(target)
 
@@ -411,10 +412,22 @@ def test_package_to_json(tmpdir):
         assert package == json.load(file)
 
 
+def test_package_to_yaml(tmpdir):
+
+    # Write
+    target = os.path.join(tmpdir, "package.yaml")
+    package = Package("data/package.json")
+    package.to_yaml(target)
+
+    # Read
+    with open(target, encoding="utf-8") as file:
+        assert package == yaml.safe_load(file)
+
+
 def test_package_to_zip(tmpdir):
 
     # Write
-    target = os.path.join(tmpdir, "datapackage.zip")
+    target = os.path.join(tmpdir, "package.zip")
     package = Package("data/package.json")
     package.to_zip(target)
 
@@ -435,7 +448,7 @@ def test_package_to_zip_source_remote(tmpdir):
 
     # Write
     path = BASE_URL % "data/table.csv"
-    target = os.path.join(tmpdir, "datapackage.zip")
+    target = os.path.join(tmpdir, "package.zip")
     package = Package(name="name", resources=[{"name": "name", "path": path}])
     package.to_zip(target)
 
@@ -454,7 +467,7 @@ def test_package_to_zip_source_remote(tmpdir):
 def test_package_to_zip_source_inline(tmpdir):
 
     # Read
-    target = os.path.join(tmpdir, "datapackage.zip")
+    target = os.path.join(tmpdir, "package.zip")
     data = [["id", "name"], ["1", "english"], ["2", "中国人"]]
     package = Package(name="name", resources=[{"name": "name", "data": data}])
     package.to_zip(target)
diff --git a/tests/test_resource.py b/tests/test_resource.py
index b17f5169..2baef210 100644
--- a/tests/test_resource.py
+++ b/tests/test_resource.py
@@ -1,5 +1,6 @@
 import os
 import json
+import yaml
 import pytest
 from frictionless import Resource, exceptions
 
@@ -585,7 +586,7 @@ def test_resource_infer_from_path():
 def test_resource_to_json(tmpdir):
 
     # Write
-    target = os.path.join(tmpdir, "dataresource.json")
+    target = os.path.join(tmpdir, "resource.json")
     resource = Resource("data/resource.json")
     resource.to_json(target)
 
@@ -594,10 +595,22 @@ def test_resource_to_json(tmpdir):
         assert resource == json.load(file)
 
 
+def test_resource_to_yaml(tmpdir):
+
+    # Write
+    target = os.path.join(tmpdir, "resource.yaml")
+    resource = Resource("data/resource.json")
+    resource.to_yaml(target)
+
+    # Read
+    with open(target, encoding="utf-8") as file:
+        assert resource == yaml.safe_load(file)
+
+
 def test_resource_to_zip(tmpdir):
 
     # Write
-    target = os.path.join(tmpdir, "dataresource.zip")
+    target = os.path.join(tmpdir, "resource.zip")
     resource = Resource("data/resource.json")
     resource.to_zip(target)
 
@@ -615,7 +628,7 @@ def test_resource_to_zip_source_remote(tmpdir):
 
     # Write
     path = BASE_URL % "data/table.csv"
-    target = os.path.join(tmpdir, "datapackage.zip")
+    target = os.path.join(tmpdir, "resource.zip")
     resource = Resource(name="name", path=path)
     resource.to_zip(target)
 
@@ -631,7 +644,7 @@ def test_resource_to_zip_source_remote(tmpdir):
 def test_resource_to_zip_source_inline(tmpdir):
 
     # Write
-    target = os.path.join(tmpdir, "dataresource.zip")
+    target = os.path.join(tmpdir, "resource.zip")
     data = [["id", "name"], ["1", "english"], ["2", "中国人"]]
     resource = Resource(name="name", data=data)
     resource.to_zip(target)
diff --git a/tests/test_schema.py b/tests/test_schema.py
index 36998a10..7ba7ffb7 100644
--- a/tests/test_schema.py
+++ b/tests/test_schema.py
@@ -1,5 +1,6 @@
 import io
 import json
+import yaml
 import pytest
 import requests
 from decimal import Decimal
@@ -334,6 +335,18 @@ def test_schema_to_json(tmpdir):
         assert schema == json.load(file)
 
 
+def test_schema_to_yaml(tmpdir):
+
+    # Write
+    target = str(tmpdir.join("schema.yaml"))
+    schema = Schema(DESCRIPTOR_MIN)
+    schema.to_yaml(target)
+
+    # Read
+    with open(target, encoding="utf-8") as file:
+        assert schema == yaml.safe_load(file)
+
+
 # Issues