Repository: cldf/csvw

Issue:
Example of programmatically creating metadata from scratch
Hello,

I was wondering if it would be possible to provide an example of creating the metadata from scratch. My goal is to create a metadata file by programmatically examining a CSV file to determine the schema. No worries if this is out of scope for the library.

Thank you!

Comments:

Patch:
diff --git a/README.md b/README.md
index a90bdd4..dc0d42b 100644
--- a/README.md
+++ b/README.md
@@ -72,14 +72,36 @@ probably never will) implement the full extent of this spec.
   description.
 
 
+## Compatibility with [Frictionless Data Specs](https://specs.frictionlessdata.io/)
+
+The CSVW-described dataset is basically equivalent to a [Frictionless DataPackage] where all [Data Resources](https://specs.frictionlessdata.io/data-resource/) are [Tabular Data](https://specs.frictionlessdata.io/tabular-data-resource/).
+Thus, the `csvw` package provides some conversion functionality. To
+"read CSVW data from a Data Package", there's the `csvw.TableGroup.from_frictionless_datapackage` method:
+```python
+from csvw import TableGroup
+tg = TableGroup.from_frictionless_datapackage('PATH/TO/datapackage.json')
+```
+To convert the metadata, the `TableGroup` can then be serialzed:
+```python
+tg.to_file('csvw-metadata.json')
+```
+
+Note that the CSVW metadata file must be written to the Data Package's directory
+to make sure relative paths to data resources work.
+
+This functionality - together with the schema inference capabilities
+of [`frictionless describe`](https://frictionlessdata.io/tooling/python/describing-data/#describe-functions) - provides
+a convenient way to bootstrap CSVW metadata for a set of "raw" CSV
+files.
+
+
 ## See also
 
 - https://www.w3.org/2013/csvw/wiki/Main_Page
 - https://github.com/CLARIAH/COW
 - https://github.com/CLARIAH/ruminator
 - https://github.com/bloomberg/pycsvw
-- https://github.com/frictionlessdata/goodtables-py
-- https://github.com/frictionlessdata/tableschema-py
+- https://specs.frictionlessdata.io/table-schema/
 - https://github.com/theodi/csvlint.rb
 - https://github.com/ruby-rdf/rdf-tabular
 - https://github.com/rdf-ext/rdf-parser-csvw
diff --git a/src/csvw/frictionless.py b/src/csvw/frictionless.py
new file mode 100644
index 0000000..eca4f22
--- /dev/null
+++ b/src/csvw/frictionless.py
@@ -0,0 +1,222 @@
+"""
+Functionality to convert tabular data in Frictionless Data Packages to CSVW.
+
+We translate [table schemas](https://specs.frictionlessdata.io/table-schema/) defined
+for [data resources](https://specs.frictionlessdata.io/data-resource/) in a
+[data package](https://specs.frictionlessdata.io/data-package/) to a CVSW TableGroup.
+
+This functionality can be used together with the `frictionless describe` command to add
+CSVW metadata to "raw" CSV tables.
+"""
+import json
+import pathlib
+
+
+def convert_column_spec(spec):
+    """
+    https://specs.frictionlessdata.io/table-schema/#field-descriptors
+
+    :param spec:
+    :return:
+    """
+    typemap = {
+        'year': 'gYear',
+        'yearmonth': 'gYearMonth',
+    }
+
+    titles = [t for t in [spec.get('title')] if t]
+
+    res = {'name': spec['name'], 'datatype': {'base': 'string'}}
+    if 'type' in spec:
+        if spec['type'] == 'string' and spec.get('format') == 'binary':
+            res['datatype']['base'] = 'binary'
+        elif spec['type'] == 'string' and spec.get('format') == 'uri':
+            res['datatype']['base'] = 'anyURI'
+        elif spec['type'] in typemap:
+            res['datatype']['base'] = typemap[spec['type']]
+        elif spec['type'] in [
+            'string', 'number', 'integer', 'boolean', 'date', 'time', 'datetime', 'duration',
+        ]:
+            res['datatype']['base'] = spec['type']
+            if spec['type'] == 'string' and spec.get('format'):
+                res['datatype']['dc:format'] = spec['format']
+            if spec['type'] == 'boolean' and spec.get('trueValues') and spec.get('falseValues'):
+                res['datatype']['format'] = '{}|{}'.format(
+                    spec['trueValues'][0], spec['falseValues'][0])
+            if spec['type'] in ['number', 'integer']:
+                if spec.get('bareNumber') is True:  # pragma: no cover
+                    raise NotImplementedError(
+                        'bareNumber is not supported in CSVW. It may be possible to translate to '
+                        'a number pattern, though. See '
+                        'https://www.w3.org/TR/2015/REC-tabular-data-model-20151217/'
+                        '#formats-for-numeric-types')
+                if any(prop in spec for prop in ['decimalChar', 'groupChar']):
+                    res['datatype']['format'] = {}
+                    for p in ['decimalChar', 'groupChar']:
+                        if spec.get(p):
+                            res['datatype']['format'][p] = spec[p]
+        elif spec['type'] in ['object', 'array']:
+            res['datatype']['base'] = 'json'
+            res['datatype']['dc:format'] = 'application/json'
+        elif spec['type'] == 'geojson':
+            res['datatype']['base'] = 'json'
+            res['datatype']['dc:format'] = 'application/geo+json'
+
+    if titles:
+        res['titles'] = titles
+    if 'description' in spec:
+        res['dc:description'] = [spec['description']]
+    if 'rdfType' in spec:
+        res['propertyUrl'] = spec['rdfType']
+
+    constraints = spec.get('constraints', {})
+    for prop in ['required', 'minLength', 'maxLength', 'minimum', 'maximum']:
+        if prop in constraints:
+            res['datatype'][prop] = constraints[prop]
+        if ('pattern' in constraints) and ('format' not in res['datatype']):
+            res['datatype']['format'] = constraints['pattern']
+        # FIXME: we could transform the "enum" constraint for string into
+        # a regular expression in the "format" property.
+    return res
+
+
+def convert_foreignKey(rsc_name, fk, resource_map):
+    """
+    https://specs.frictionlessdata.io/table-schema/#foreign-keys
+    """
+    # Rename "fields" to "columnReference" and map resource name to url (resolving self-referential
+    # foreign keys).
+    return dict(
+        columnReference=fk['fields'],
+        reference=dict(
+            columnReference=fk['reference']['fields'],
+            resource=resource_map[fk['reference']['resource'] or rsc_name],
+        )
+    )
+
+
+def convert_table_schema(rsc_name, schema, resource_map):
+    """
+    :param rsc_name: `name` property of the resource the schema belongs to. Needed to resolve \
+    self-referential foreign keys.
+    :param schema: `dict` parsed from JSON representing a frictionless Table Schema object.
+    :param resource_map: `dict` mapping resource names to resource paths, needed to convert foreign\
+    key constraints.
+    :return: `dict` suitable for instantiating a `csvw.metadata.Schema` object.
+    """
+    res = dict(
+        columns=[convert_column_spec(f) for f in schema['fields']],
+    )
+    for prop in [
+        ('missingValues', 'null'),
+        'primaryKey',
+        'foreignKeys',
+    ]:
+        if isinstance(prop, tuple):
+            prop, toprop = prop
+        else:
+            toprop = prop
+        if prop in schema:
+            res[toprop] = schema[prop]
+            if prop == 'foreignKeys':
+                res[toprop] = [convert_foreignKey(rsc_name, fk, resource_map) for fk in res[toprop]]
+    return res
+
+
+def convert_dialect(rsc):
+    """
+    https://specs.frictionlessdata.io/csv-dialect/
+    """
+    d = rsc.get('dialect', {})
+    res = {}
+    if d.get('delimiter'):
+        res['delimiter'] = d['delimiter']
+    if rsc.get('encoding'):
+        res['encoding'] = rsc['encoding']
+    for prop in [
+        'delimiter',
+        'quoteChar',
+        'doubleQuote',
+        'skipInitialSpace',
+        'header',
+    ]:
+        if prop in d:
+            res[prop] = d[prop]
+    if 'lineTerminator' in d:
+        res['lineTerminators'] = [d['lineTerminator']]
+    if 'commentChar' in d:
+        res['commentPrefix'] = d['commentChar']
+    return res
+
+
+class DataPackage:
+    def __init__(self, spec, directory=None):
+        if isinstance(spec, DataPackage):
+            self.json = spec.json
+            self.dir = spec.dir
+            return
+        if isinstance(spec, dict):
+            # already a parsed JSON object
+            self.dir = pathlib.Path(directory or '.')
+        elif isinstance(spec, pathlib.Path):
+            self.dir = directory or spec.parent
+            spec = json.loads(spec.read_text(encoding='utf8'))
+        else:  # assume a JSON formatted string
+            spec = json.loads(spec)
+            self.dir = pathlib.Path(directory or '.')
+
+        self.json = spec
+
+    def to_tablegroup(self, cls=None):
+        from csvw import TableGroup
+
+        md = {}
+        # Package metadata:
+        md['dc:replaces'] = json.dumps(self.json)
+
+        # version,
+        # image,
+
+        for flprop, csvwprop in [
+            ('id', 'dc:identifier'),
+            ('licenses', 'dc:license'),
+            ('title', 'dc:title'),
+            ('homepage', 'dcat:accessURL'),
+            ('description', 'dc:description'),
+            ('sources', 'dc:source'),
+            ('contributors', 'dc:contributor'),
+            ('profile', 'dc:conformsTo'),
+            ('keywords', 'dc:subject'),
+            ('created', 'dc:created'),
+        ]:
+            if flprop in self.json:
+                md[csvwprop] = self.json[flprop]
+
+        if 'name' in self.json:
+            if 'id' not in self.json:
+                md['dc:identifier'] = self.json['name']
+            elif 'title' not in self.json:
+                md['dc:title'] = self.json['name']
+
+        # Data Resource metadata:
+        resources = [rsc for rsc in self.json.get('resources', []) if 'path' in rsc]
+        resource_map = {rsc['name']: rsc['path'] for rsc in resources if 'name' in rsc}
+        for rsc in resources:
+            schema = rsc.get('schema')
+            if schema and \
+                    rsc.get('profile') == 'tabular-data-resource' and \
+                    rsc.get('scheme') == 'file' and \
+                    rsc.get('format') == 'csv':
+                # Table Schema:
+                md.setdefault('tables', [])
+                table = dict(
+                    url=rsc['path'],
+                    tableSchema=convert_table_schema(rsc.get('name'), schema, resource_map),
+                    dialect=convert_dialect(rsc),
+                )
+                md['tables'].append(table)
+
+        cls = cls or TableGroup
+        res = cls.fromvalue(md)
+        res._fname = self.dir / 'csvw-metadata.json'
+        return res
diff --git a/src/csvw/metadata.py b/src/csvw/metadata.py
index 4c69354..9c71fe8 100644
--- a/src/csvw/metadata.py
+++ b/src/csvw/metadata.py
@@ -8,7 +8,6 @@ This module implements (partially) the W3C recommendation
 .. seealso:: https://www.w3.org/TR/tabular-metadata/
 """
 import io
-import re
 import json
 import shutil
 import pathlib
@@ -27,6 +26,7 @@ import uritemplate
 from . import utils
 from .datatypes import DATATYPES
 from .dsv import Dialect, UnicodeReaderWithLineNumber, UnicodeWriter
+from .frictionless import DataPackage
 
 DEFAULT = object()
 
@@ -38,11 +38,6 @@ __all__ = [
 ]
 
 
-# Level 1 variable names according to https://tools.ietf.org/html/rfc6570#section-2.3:
-_varchar = r'([a-zA-Z0-9_]|\%[a-fA-F0-9]{2})'
-_varname = re.compile('(' + _varchar + '([.]?' + _varchar + ')*)$')
-
-
 def log_or_raise(msg, log=None, level='warning', exception_cls=ValueError):
     if log:
         getattr(log, level)(msg)
@@ -368,9 +363,7 @@ class Description(DescriptionBase):
 @attr.s
 class Column(Description):
 
-    name = attr.ib(
-        default=None,
-        validator=utils.attr_valid_re(_varname, nullable=True))
+    name = attr.ib(default=None)
     suppressOutput = attr.ib(default=False)
     titles = attr.ib(
         default=None,
@@ -783,6 +776,10 @@ class TableGroup(TableLike):
         for table in self.tables:
             table._parent = self
 
+    @classmethod
+    def from_frictionless_datapackage(cls, dp):
+        return DataPackage(dp).to_tablegroup(cls)
+
     def read(self):
         """
         Read all data of a TableGroup
diff --git a/src/csvw/utils.py b/src/csvw/utils.py
index 6cf85a0..c423732 100644
--- a/src/csvw/utils.py
+++ b/src/csvw/utils.py
@@ -30,26 +30,6 @@ def attr_asdict(obj, omit_defaults=True, omit_private=True):
     return res
 
 
-def attr_valid_re(regex_or_pattern, nullable=False):
-    if hasattr(regex_or_pattern, 'match'):
-        pattern = regex_or_pattern
-    else:
-        pattern = re.compile(regex_or_pattern)
-
-    msg = '{0} is not a valid {1}'
-
-    if nullable:
-        def valid_re(instance, attribute, value):
-            if value is not None and pattern.match(value) is None:
-                raise ValueError(msg.format(value, attribute.name))
-    else:
-        def valid_re(instance, attribute, value):
-            if pattern.match(value) is None:
-                raise ValueError(msg.format(value, attribute.name))
-
-    return valid_re
-
-
 class lazyproperty(object):
     """Non-data descriptor caching the computed result as instance attribute.
     >>> import itertools


Test patch:
diff --git a/tests/fixtures/datapackage.json b/tests/fixtures/datapackage.json
new file mode 100644
index 0000000..57c9221
--- /dev/null
+++ b/tests/fixtures/datapackage.json
@@ -0,0 +1,59 @@
+{
+  "profile": "data-package",
+  "resources": [
+    {
+      "path": "frictionless-data.csv",
+      "stats": {
+        "hash": "b36e8c21563ab32645052c11510bddb7", "bytes": 131, "fields": 3, "rows": 9
+      },
+      "control": {"newline": ""},
+      "encoding": "utf-8",
+      "dialect": {"delimiter": "|", "commentChar":  "#", "lineTerminator":  "+\n"},
+      "schema": {
+        "fields": [
+          {"name":  "FK", "type":  "string"},
+          {"name": "Year", "type": "integer", "rdfType":  "http://example.com"},
+          {"name": "Location name", "type": "string"},
+          {"name": "Value", "type": "integer", "groupChar":  "-"},
+          {"name":  "binary", "type":  "string", "format":  "binary"},
+          {"name":  "anyURI", "type":  "string", "format":  "uri"},
+          {"name":  "email", "type":  "string", "format":  "email"},
+          {"name":  "boolean", "type":  "boolean", "trueValues":  ["ja"], "falseValues":  ["nein"]},
+          {"name": "array", "type": "array", "description":  "empty"},
+          {"name": "geojson", "type": "geojson", "title":  "a point"}
+        ],
+        "foreignKeys": [
+          {"fields":  ["FK"], "reference":  {"resource": "tsv", "fields":  ["class"]}}
+        ]
+      },
+      "name": "test",
+      "profile": "tabular-data-resource",
+      "scheme": "file",
+      "format": "csv",
+      "hashing": "md5",
+      "compression": "no",
+      "compressionPath": "",
+      "query": {}
+    },
+    {
+      "path": "tsv.txt",
+      "control": {"newline": ""},
+      "encoding": "utf-8",
+      "dialect": {"delimiter": "\t"},
+      "schema": {
+        "fields": [
+          {"name": "class", "type": "string"},
+          {"name": "a-name", "type": "string"}
+        ]
+      },
+      "name": "tsv",
+      "profile": "tabular-data-resource",
+      "scheme": "file",
+      "format": "csv",
+      "hashing": "md5",
+      "compression": "no",
+      "compressionPath": "",
+      "query": {}
+    }
+  ]
+}
diff --git a/tests/fixtures/frictionless-data.csv b/tests/fixtures/frictionless-data.csv
new file mode 100644
index 0000000..f931adf
--- /dev/null
+++ b/tests/fixtures/frictionless-data.csv
@@ -0,0 +1,11 @@
+FK|Year|Location name|Value|binary|anyURI|email|boolean|array|geojson+
+a|2010||10-123|eA==|http://example.com|x@example.com|ja|[]|{}+
+a|2011||20||||nein||+
+c|2012||30||||||+
+#+
+e|2010|Urban|12||||||+
+e|2011|Urban|22||||||+
+e|2012|Urban|32||||||+
+e|2010|Rural|14||||||+
+e|2011|Rural|24||||||+
+x|2012|Rural|34||||||+
diff --git a/tests/test_frictionless.py b/tests/test_frictionless.py
new file mode 100644
index 0000000..150bdf1
--- /dev/null
+++ b/tests/test_frictionless.py
@@ -0,0 +1,84 @@
+import json
+import shutil
+import pathlib
+
+import pytest
+
+from csvw.dsv import UnicodeWriter
+from csvw import TableGroup
+from csvw.frictionless import DataPackage
+
+FIXTURES = pathlib.Path(__file__).parent / 'fixtures'
+
+
+@pytest.fixture
+def tmpfixtures(tmpdir):
+    shutil.copytree(str(pathlib.Path(__file__).parent / 'fixtures'), str(tmpdir.join('fixtures')))
+    return pathlib.Path(str(tmpdir)) / 'fixtures'
+
+
+@pytest.fixture
+def datafactory(tmpdir):
+    def make(fields, data):
+        p = pathlib.Path(str(tmpdir)) / 'datapackage.json'
+        with p.open(mode='wt') as f:
+            rsc = dict(
+                profile='tabular-data-resource',
+                scheme='file',
+                format='csv',
+                path='data.csv',
+                schema=dict(fields=fields),
+            )
+            json.dump(dict(resources=[rsc]), f)
+        with UnicodeWriter(p.parent / 'data.csv') as w:
+            w.writerow([f['name'] for f in fields])
+            w.writerows(data)
+        return p
+    return make
+
+
+def test_DataPackage_init():
+    dp = DataPackage(dict(resources=[], name='x'))
+    dp = DataPackage(dp)
+    assert dp.to_tablegroup().common_props['dc:identifier'] == 'x'
+    dp = DataPackage('{"resources": [], "name": "x", "id": "y"}')
+    assert dp.to_tablegroup().common_props['dc:identifier'] == 'y'
+    assert dp.to_tablegroup().common_props['dc:title'] == 'x'
+
+
+def test_DataPackage_constraints(datafactory):
+    dp = datafactory([{'name': 'col', 'constraints': {'maxLength': 3}}], [['abcd']])
+    with pytest.raises(ValueError):
+        _ = list(DataPackage(dp).to_tablegroup().tables[0])
+
+    dp = datafactory([{'name': 'col', 'constraints': {'pattern': '[a-z]{2}'}}], [['abcd']])
+    with pytest.raises(ValueError):
+        _ = list(DataPackage(dp).to_tablegroup().tables[0])
+
+    dp = datafactory(
+        [{'name': 'col', 'type': 'year', 'constraints': {'pattern': '[2].*'}}], [['1990']])
+    with pytest.raises(ValueError):
+        _ = list(DataPackage(dp).to_tablegroup().tables[0])
+
+
+def test_DataPackage(tmpfixtures):
+    dp = DataPackage(tmpfixtures / 'datapackage.json')
+    tg = dp.to_tablegroup()
+    rows = list(tg.tables[0])
+    assert len(rows) == 9
+    assert rows[-1]['Year'] == 2012
+    assert rows[-1]['Location name'] == 'Rural'
+    with pytest.raises(ValueError):
+        tg.check_referential_integrity()
+    schema = tg.tables[0].tableSchema
+    for c in ['binary', 'anyURI']:
+        assert schema.columndict[c].datatype.base == c
+    assert rows[0]['boolean'] is True and rows[1]['boolean'] is False
+    assert rows[0]['Value'] == 10123
+    assert list(rows[0].values())[-1] != '+', "custom line terminator must be stripped"
+
+    tg.to_file(tmpfixtures / 'metadata.json')
+    tg = TableGroup.from_file(tmpfixtures / 'metadata.json')
+    rows = list(tg.tables[0])
+    assert len(rows) == 9
+    assert rows[-1]['Year'] == 2012
diff --git a/tests/test_metadata.py b/tests/test_metadata.py
index c840947..7f461fb 100644
--- a/tests/test_metadata.py
+++ b/tests/test_metadata.py
@@ -29,12 +29,6 @@ def test_Link():
     assert li.resolve(pathlib.Path('.')) == pathlib.Path('abc.csv')
 
 
-def test_column_init():
-    with pytest.raises(ValueError):
-        # column names mustn't start with a -!
-        csvw.Column(name='-abd')
-
-
 class TestColumnAccess(object):
 
     def test_get_column(self):
@@ -279,6 +273,10 @@ class TestTableGroup(object):
         return _make_table_like(
             csvw.TableGroup, tmpdir, data=data, metadata=metadata, mdname='csv.txt-metadata.json')
 
+    def test_from_frictionless(self):
+        tg = csvw.TableGroup.from_frictionless_datapackage(FIXTURES / 'datapackage.json')
+        assert list(tg.tables[0])
+
     def test_iteritems_column_renaming(self, tmpdir):
         t = csvw.TableGroup.from_file(FIXTURES / 'test.tsv-metadata.json')
         items = list(t.tables[0])
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 20e88b0..0fc3956 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -1,14 +1,6 @@
-import pytest
-
 from csvw import utils
 
 
-def test_attr_valid_re(mocker):
-    validator = utils.attr_valid_re('[0-9]+')
-    with pytest.raises(ValueError):
-        validator(None, mocker.Mock(), 'x')
-
-
 def test_lazyproperty():
     import itertools