Repository: cldf/csvw

Issue:
Add support for reading metadata and data from URLs

Discussion:

Patch:
diff --git a/src/csvw/frictionless.py b/src/csvw/frictionless.py
index eca4f22..2341419 100644
--- a/src/csvw/frictionless.py
+++ b/src/csvw/frictionless.py
@@ -125,6 +125,8 @@ def convert_table_schema(rsc_name, schema, resource_map):
 
 def convert_dialect(rsc):
     """
+    Limitations: lineTerminator is not supported.
+
     https://specs.frictionlessdata.io/csv-dialect/
     """
     d = rsc.get('dialect', {})
@@ -142,8 +144,6 @@ def convert_dialect(rsc):
     ]:
         if prop in d:
             res[prop] = d[prop]
-    if 'lineTerminator' in d:
-        res['lineTerminators'] = [d['lineTerminator']]
     if 'commentChar' in d:
         res['commentPrefix'] = d['commentChar']
     return res
diff --git a/src/csvw/metadata.py b/src/csvw/metadata.py
index 9c71fe8..5d0dda6 100644
--- a/src/csvw/metadata.py
+++ b/src/csvw/metadata.py
@@ -8,6 +8,7 @@ This module implements (partially) the W3C recommendation
 .. seealso:: https://www.w3.org/TR/tabular-metadata/
 """
 import io
+import re
 import json
 import shutil
 import pathlib
@@ -17,7 +18,7 @@ import warnings
 import itertools
 import contextlib
 import collections
-from urllib.parse import urljoin
+from urllib.parse import urljoin, urlparse, urlunparse
 from urllib.request import urlopen
 
 import attr
@@ -35,9 +36,14 @@ __all__ = [
     'Table', 'Column', 'ForeignKey',
     'Link', 'NaturalLanguage',
     'Datatype',
+    'is_url',
 ]
 
 
+def is_url(s):
+    return re.match(r'https?://', str(s))
+
+
 def log_or_raise(msg, log=None, level='warning', exception_cls=ValueError):
     if log:
         getattr(log, level)(msg)
@@ -171,7 +177,7 @@ class NaturalLanguage(collections.OrderedDict):
 class DescriptionBase(object):
     """Container for
     - common properties (see http://w3c.github.io/csvw/metadata/#common-properties)
-    - @-properies.
+    - @-properties.
     """
 
     common_props = attr.ib(default=attr.Factory(dict))
@@ -538,6 +544,8 @@ class TableLike(Description):
     @classmethod
     def from_file(cls, fname):
         if not isinstance(fname, pathlib.Path):
+            if is_url(fname):
+                return cls.from_url(str(fname))
             fname = pathlib.Path(fname)
         with json_open(str(fname)) as f:
             data = json.load(f)
@@ -545,6 +553,16 @@ class TableLike(Description):
         res._fname = fname
         return res
 
+    @classmethod
+    def from_url(cls, url):
+        with io.TextIOWrapper(urlopen(url), encoding='utf8') as f:
+            data = json.load(f)
+        res = cls.fromvalue(data)
+        if 'base' not in res.at_props:
+            url = urlparse(url)
+            res.at_props['base'] = urlunparse((url.scheme, url.netloc, url.path, '', '', ''))
+        return res
+
     def to_file(self, fname, omit_defaults=True):
         if not isinstance(fname, pathlib.Path):
             fname = pathlib.Path(fname)
@@ -558,6 +576,9 @@ class TableLike(Description):
         """
         We only support data in the filesystem, thus we make sure `base` is a `pathlib.Path`.
         """
+        at_props = self._parent.at_props if self._parent else self.at_props
+        if 'base' in at_props:
+            return at_props['base']
         return self._parent._fname.parent if self._parent else self._fname.parent
 
 
@@ -682,15 +703,18 @@ class Table(TableLike):
                 requiredcols.add(col.header)
 
         with contextlib.ExitStack() as stack:
-            handle = fname
-            fpath = pathlib.Path(fname)
-            if not fpath.exists():
-                zipfname = fpath.parent.joinpath(fpath.name + '.zip')
-                if zipfname.exists():
-                    zipf = stack.enter_context(zipfile.ZipFile(str(zipfname)))
-                    handle = io.TextIOWrapper(
-                        zipf.open([n for n in zipf.namelist() if n.endswith(fpath.name)][0]),
-                        encoding=dialect.encoding)
+            if is_url(fname):
+                handle = io.TextIOWrapper(urlopen(str(fname)), encoding=dialect.encoding)
+            else:
+                handle = fname
+                fpath = pathlib.Path(fname)
+                if not fpath.exists():
+                    zipfname = fpath.parent.joinpath(fpath.name + '.zip')
+                    if zipfname.exists():
+                        zipf = stack.enter_context(zipfile.ZipFile(str(zipfname)))
+                        handle = io.TextIOWrapper(
+                            zipf.open([n for n in zipf.namelist() if n.endswith(fpath.name)][0]),
+                            encoding=dialect.encoding)
 
             reader = stack.enter_context(UnicodeReaderWithLineNumber(handle, dialect=dialect))
             reader = iter(reader)


Test patch:
diff --git a/tests/fixtures/datapackage.json b/tests/fixtures/datapackage.json
index 57c9221..308a4f3 100644
--- a/tests/fixtures/datapackage.json
+++ b/tests/fixtures/datapackage.json
@@ -8,7 +8,7 @@
       },
       "control": {"newline": ""},
       "encoding": "utf-8",
-      "dialect": {"delimiter": "|", "commentChar":  "#", "lineTerminator":  "+\n"},
+      "dialect": {"delimiter": "|", "commentChar":  "#", "lineTerminator":  "\n"},
       "schema": {
         "fields": [
           {"name":  "FK", "type":  "string"},
diff --git a/tests/fixtures/frictionless-data.csv b/tests/fixtures/frictionless-data.csv
index f931adf..1e40e59 100644
--- a/tests/fixtures/frictionless-data.csv
+++ b/tests/fixtures/frictionless-data.csv
@@ -1,11 +1,11 @@
-FK|Year|Location name|Value|binary|anyURI|email|boolean|array|geojson+
-a|2010||10-123|eA==|http://example.com|x@example.com|ja|[]|{}+
-a|2011||20||||nein||+
-c|2012||30||||||+
-#+
-e|2010|Urban|12||||||+
-e|2011|Urban|22||||||+
-e|2012|Urban|32||||||+
-e|2010|Rural|14||||||+
-e|2011|Rural|24||||||+
-x|2012|Rural|34||||||+
+FK|Year|Location name|Value|binary|anyURI|email|boolean|array|geojson
+a|2010||10-123|eA==|http://example.com|x@example.com|ja|[]|{}
+a|2011||20||||nein||
+c|2012||30||||||
+#
+e|2010|Urban|12||||||
+e|2011|Urban|22||||||
+e|2012|Urban|32||||||
+e|2010|Rural|14||||||
+e|2011|Rural|24||||||
+x|2012|Rural|34||||||
diff --git a/tests/test_frictionless.py b/tests/test_frictionless.py
index 150bdf1..4fbfea4 100644
--- a/tests/test_frictionless.py
+++ b/tests/test_frictionless.py
@@ -75,7 +75,6 @@ def test_DataPackage(tmpfixtures):
         assert schema.columndict[c].datatype.base == c
     assert rows[0]['boolean'] is True and rows[1]['boolean'] is False
     assert rows[0]['Value'] == 10123
-    assert list(rows[0].values())[-1] != '+', "custom line terminator must be stripped"
 
     tg.to_file(tmpfixtures / 'metadata.json')
     tg = TableGroup.from_file(tmpfixtures / 'metadata.json')
diff --git a/tests/test_metadata.py b/tests/test_metadata.py
index 7f461fb..fc85add 100644
--- a/tests/test_metadata.py
+++ b/tests/test_metadata.py
@@ -775,3 +775,12 @@ def test_zip_support(tmpdir):
 
     tg.write(out.parent / 'md.json', _zipped=True, **{'zipped.csv': res + res})
     assert len(list(csvw.TableGroup.from_file(out.parent / 'md.json').tables[0])) == 4
+
+
+def test_from_url(mocker):
+    from io import BytesIO
+    mocker.patch(
+        'csvw.metadata.urlopen',
+        lambda u: BytesIO(FIXTURES.joinpath(u.split('/')[-1]).read_bytes()))
+    t = csvw.Table.from_file('http://example.com/csv.txt-table-metadata.json')
+    assert len(list(t)) == 2