Repository: altair-viz/altair
Pull request #372 (2017-08-09 19:16:33)

Schema loc
Addresses #370; Built on top of #364 – don't merge until after #364 is merged (may need a rebase)


Patch:
diff --git a/altair/api.py b/altair/api.py
index b95559b3..9628f750 100644
--- a/altair/api.py
+++ b/altair/api.py
@@ -150,6 +150,7 @@ class Transform(schema.Transform):
 # Top-level Objects
 #*************************************************************************
 class TopLevelMixin(object):
+
     @staticmethod
     def _png_output_available():
         return node.vl_cmd_available('vl2png')
@@ -251,7 +252,9 @@ class TopLevelMixin(object):
         spec : dict
             The JSON specification of the chart object.
         """
-        return super(TopLevelMixin, self).to_dict(data=data)
+        dct = super(TopLevelMixin, self).to_dict(data=data)
+        dct['$schema'] = schema.vegalite_schema_url
+        return dct
 
     @classmethod
     def from_dict(cls, dct):
@@ -267,6 +270,14 @@ class TopLevelMixin(object):
         chart : Chart object
             The altair Chart object built from the specification.
         """
+        if '$schema' in dct:
+            if dct['$schema'] != schema.vegalite_schema_url:
+                warnings.warn('from_dict: $schema={0} does not match '
+                              'schema used to build this Altair version '
+                              '({1}. '
+                              ''.format(dct['$schema'],
+                                        schema.vegalite_schema_url))
+            dct = {k: v for k, v in dct.items() if k != '$schema'}
         return super(TopLevelMixin, cls).from_dict(dct)
 
     def to_json(self, data=True, sort_keys=True, **kwargs):
diff --git a/altair/expr/core.py b/altair/expr/core.py
index ceb2ff12..ff502cac 100644
--- a/altair/expr/core.py
+++ b/altair/expr/core.py
@@ -221,6 +221,7 @@ class DataFrame(object):
     >>> chart = Chart(df)
     >>> print(chart.to_json(indent=2))         # doctest: +NORMALIZE_WHITESPACE
     {
+      "$schema": "https://vega.github.io/schema/vega-lite/v1.2.1.json",
       "data": {
         "url": "url/to/my/data.json"
       },
diff --git a/altair/schema/__init__.py b/altair/schema/__init__.py
index 22de4f72..e7f4acf5 100644
--- a/altair/schema/__init__.py
+++ b/altair/schema/__init__.py
@@ -1,6 +1,6 @@
 from ._interface import *
 from ._schema import SCHEMA_FILE, load_schema
-from ._vegalite_version import vegalite_version
+from ._vegalite_version import vegalite_version, vegalite_schema_url
 from . import visitors
 
 from ._interface import jstraitlets
diff --git a/altair/schema/_interface/__init__.py b/altair/schema/_interface/__init__.py
index 3ab20347..0b7164a1 100644
--- a/altair/schema/_interface/__init__.py
+++ b/altair/schema/_interface/__init__.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Auto-generated by schemapi: do not modify file directly
-# - schemapi version: 0.3.0.dev0+414a076
-# - date:    2017-08-07 17:41:13
+# - schemapi version: 0.3.0.dev0+ee22edf
+# - date:    2017-08-09 12:14:26
 
 from .schema import Root
 from .schema import AggregateOp
diff --git a/altair/schema/_interface/channel_collections.py b/altair/schema/_interface/channel_collections.py
index a3ce4d2e..18899869 100644
--- a/altair/schema/_interface/channel_collections.py
+++ b/altair/schema/_interface/channel_collections.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Auto-generated file: do not modify directly
-# - altair version info: v1.2.0-96-g99cfe91
-# - date: 2017-08-07 17:41:14
+# - altair version info: v1.2.0-98-g8a98636
+# - date: 2017-08-09 12:14:27
 
 import traitlets as T
 from . import jstraitlets as jst
diff --git a/altair/schema/_interface/channel_wrappers.py b/altair/schema/_interface/channel_wrappers.py
index fa24589d..411f4a74 100644
--- a/altair/schema/_interface/channel_wrappers.py
+++ b/altair/schema/_interface/channel_wrappers.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Auto-generated file: do not modify directly
-# - altair version info: v1.2.0-96-g99cfe91
-# - date: 2017-08-07 17:41:14
+# - altair version info: v1.2.0-98-g8a98636
+# - date: 2017-08-09 12:14:26
 
 import pandas as pd
 
diff --git a/altair/schema/_interface/jstraitlets.py b/altair/schema/_interface/jstraitlets.py
index 72a5394e..eead5d05 100644
--- a/altair/schema/_interface/jstraitlets.py
+++ b/altair/schema/_interface/jstraitlets.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Auto-generated by schemapi: do not modify file directly
-# - schemapi version: 0.3.0.dev0+414a076
-# - date:    2017-08-07 17:41:13
+# - schemapi version: 0.3.0.dev0+ee22edf
+# - date:    2017-08-09 12:14:26
 
 """
 Extensions to traitlets for compatibility with JSON Schema
@@ -41,6 +41,7 @@ class UndefinedType(object):
     def __repr__(self):
         return "undefined"
 
+
 undefined = UndefinedType()
 
 
@@ -65,7 +66,14 @@ class JSONHasTraits(T.HasTraits):
     _required_traits = []  # traits required at export. If undefined, a traiterror will be raised
     _converter_registry = {}  # converter classes to use for to_dict, from_dict
 
+    # Metadata is meant to hold top-level metadata keys used in JSON Schema,
+    # for example '$schema' and/or '$id'
+    _metadata = {'$schema': undefined, '$id': undefined}
+
     def __init__(self, **kwargs):
+        # make a copy of the _metadata so we can modify locally
+        self._metadata = self._metadata.copy()
+
         # Add default traits if needed
         default = self._get_additional_traits()
         # TODO: protect against overwriting class attributes defined above.
@@ -664,6 +672,9 @@ class ToDict(Visitor):
             elif key in obj._required_traits:
                 raise T.TraitError("Required trait '{0}' is undefined'"
                                    "".format(key))
+        for key, val in obj._metadata.items():
+            if val is not undefined:
+                dct[key] = val
         return dct
 
 
@@ -698,12 +709,21 @@ class FromDict(Visitor):
             obj = cls('')
         additional_traits = cls._get_additional_traits()
 
+        # Extract metadata, if it exists
+        obj._metadata.update({prop: dct[prop]
+                              for prop in obj._metadata
+                              if prop in dct})
+
+        # Extract all other items, assigning to appropriate trait
         for prop, val in dct.items():
+            if prop in obj._metadata:
+                continue
             subtrait = obj.traits().get(prop, additional_traits)
             if not subtrait:
                 raise T.TraitError("trait {0} not valid in class {1}"
                                    "".format(prop, cls))
             obj.set_trait(prop, self.visit(subtrait, val, *args, **kwargs))
+
         return obj
 
     def visit_Instance(self, trait, dct, *args, **kwargs):
diff --git a/altair/schema/_interface/named_channels.py b/altair/schema/_interface/named_channels.py
index 8e72d4fc..d2d7c77e 100644
--- a/altair/schema/_interface/named_channels.py
+++ b/altair/schema/_interface/named_channels.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Auto-generated file: do not modify directly
-# - altair version info: v1.2.0-96-g99cfe91
-# - date: 2017-08-07 17:41:14
+# - altair version info: v1.2.0-98-g8a98636
+# - date: 2017-08-09 12:14:26
 
 from . import channel_wrappers
 
diff --git a/altair/schema/_interface/schema.py b/altair/schema/_interface/schema.py
index 8f2a81ab..084e5883 100644
--- a/altair/schema/_interface/schema.py
+++ b/altair/schema/_interface/schema.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Auto-generated by schemapi: do not modify file directly
-# - schemapi version: 0.3.0.dev0+414a076
-# - date:    2017-08-07 17:41:13
+# - schemapi version: 0.3.0.dev0+ee22edf
+# - date:    2017-08-09 12:14:26
 
 
 
diff --git a/altair/schema/_vegalite_version.py b/altair/schema/_vegalite_version.py
index 9bd431d4..5b66484d 100644
--- a/altair/schema/_vegalite_version.py
+++ b/altair/schema/_vegalite_version.py
@@ -1,4 +1,4 @@
-
 # Automatically written by tools/sync_vegalite.py
 # Do not modify this manually
-vegalite_version = '1.2.1'
+vegalite_version = 'v1.2.1'
+vegalite_schema_url = 'https://vega.github.io/schema/vega-lite/v1.2.1.json'
diff --git a/tools/sync_vegalite.py b/tools/sync_vegalite.py
index a1256d3a..0032439a 100644
--- a/tools/sync_vegalite.py
+++ b/tools/sync_vegalite.py
@@ -8,24 +8,33 @@ from contextlib import contextmanager
 
 import git  # pip install gitpython
 
+try:
+    from urllib.request import urlopen
+except ImportError:
+    # Python 2.X
+    from urllib2 import urlopen
+
 
 def abspath(*args):
     """Get absolute path relative to this file"""
     return os.path.abspath(os.path.join(os.path.dirname(__file__), *args))
 
 
-VEGALITE_TAG = 'v1.2.1'
+SCHEMA_URL = "https://vega.github.io/schema/{library}/{version}.json"
+
+VEGALITE_VERSION = 'v1.2.1'
+VEGA_VERSION = 'v2.6.5'
 VEGALITE_PATH = abspath('repos', 'vega-lite')
 VEGALITE_URL = 'https://github.com/vega/vega-lite.git'
 
-VEGA_DATA_TAG = 'v1.8.0'
+VEGA_DATA_VERSION = 'v1.8.0'
 VEGA_DATA_PATH = abspath('repos', 'vega-datasets')
 VEGA_DATA_URL = 'https://github.com/vega/vega-datasets.git'
 
-VEGALITE_VERSION_FILE = """
-# Automatically written by tools/sync_vegalite.py
+VERSION_FILE = """# Automatically written by tools/sync_vegalite.py
 # Do not modify this manually
-vegalite_version = '{version}'
+vegalite_version = '{vegalite_version}'
+vegalite_schema_url = '{vegalite_schema_url}'
 """
 
 
@@ -55,19 +64,28 @@ def checkout_tag(tag, local_repo, url):
         vegalite.branches[0].checkout()
 
 
-def sync_schema(vegalite_repo, version,
-                filename='vega-lite-schema.json',
-                destination=abspath('..', 'altair', 'schema')):
-    versionfile = os.path.join(destination, '_vegalite_version.py')
-    with open(versionfile, 'w') as f:
-        f.write(VEGALITE_VERSION_FILE.format(version=version))
+def sync_schema(vegalite_version,
+                destination=abspath('..', 'altair', 'schema'),
+                schema_filename='vega-lite-schema.json'):
+    vegalite_schema_url = SCHEMA_URL.format(library='vega-lite',
+                                            version=vegalite_version)
 
-    source = os.path.join(vegalite_repo, filename)
-    destination = os.path.join(destination, filename)
-    print("Copying Vega-Lite Schema:")
-    print("  Source:     ", source)
-    print("  Destination:", destination)
-    shutil.copyfile(source, destination)
+    version_file_path = os.path.join(destination, '_vegalite_version.py')
+    version_file_content = VERSION_FILE.format(
+                                    vegalite_version=vegalite_version,
+                                    vegalite_schema_url=vegalite_schema_url)
+    print("Writing version info")
+    print(" Filename: {0}".format(version_file_path))
+    with open(version_file_path, 'w') as f:
+        f.write(version_file_content)
+
+    schema_destination = os.path.join(destination, schema_filename)
+    print("Downloading Vega-Lite {0} Schema".format(vegalite_version))
+    print(" Source:      {0}".format(vegalite_schema_url))
+    print(" Destination: {0}".format(schema_destination))
+    with urlopen(vegalite_schema_url) as source:
+        with open(schema_destination, 'wb') as dest:
+            dest.write(source.read())
 
 
 def sync_examples(vegalite_repo,
@@ -146,12 +164,11 @@ def main():
         parser.print_help()
 
     if args.schema:
-        tag = args.tag or VEGALITE_TAG
-        with checkout_tag(tag, VEGALITE_PATH, VEGALITE_URL):
-            sync_schema(VEGALITE_PATH, tag.strip('v'))
+        version = args.tag or VEGALITE_VERSION
+        sync_schema(version)
 
     if args.examples:
-        tag = args.tag or VEGALITE_TAG
+        tag = args.tag or VEGALITE_VERSION
         with checkout_tag(tag, VEGALITE_PATH, VEGALITE_URL):
             sync_examples(VEGALITE_PATH)
 


Test patch:
diff --git a/altair/examples/tests/test_examples.py b/altair/examples/tests/test_examples.py
index 59aba24c..745fda94 100644
--- a/altair/examples/tests/test_examples.py
+++ b/altair/examples/tests/test_examples.py
@@ -20,12 +20,17 @@ def test_json_examples_round_trip(example):
 
     v = Chart.from_dict(json_dict)
     v_dict = v.to_dict()
+    if '$schema' not in json_dict:
+        v_dict.pop('$schema')
     assert v_dict == json_dict
 
     # code generation discards empty function calls, and so we
     # filter these out before comparison
     v2 = eval(v.to_python())
-    assert v2.to_dict() == remove_empty_fields(json_dict)
+    v2_dict = v2.to_dict()
+    if '$schema' not in json_dict:
+        v2_dict.pop('$schema')
+    assert v2_dict == remove_empty_fields(json_dict)
 
 
 def test_load_example():
diff --git a/altair/schema/_interface/tests/__init__.py b/altair/schema/_interface/tests/__init__.py
index 3fac123b..9909f105 100644
--- a/altair/schema/_interface/tests/__init__.py
+++ b/altair/schema/_interface/tests/__init__.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
 # Auto-generated by schemapi: do not modify file directly
-# - schemapi version: 0.3.0.dev0+414a076
-# - date:    2017-08-07 17:41:13
+# - schemapi version: 0.3.0.dev0+ee22edf
+# - date:    2017-08-09 12:14:26
 
diff --git a/altair/schema/_interface/tests/test_jstraitlets.py b/altair/schema/_interface/tests/test_jstraitlets.py
index a1503c82..1530d2d1 100644
--- a/altair/schema/_interface/tests/test_jstraitlets.py
+++ b/altair/schema/_interface/tests/test_jstraitlets.py
@@ -1,7 +1,7 @@
 # -*- coding: utf-8 -*-
 # Auto-generated by schemapi: do not modify file directly
-# - schemapi version: 0.3.0.dev0+414a076
-# - date:    2017-08-07 17:41:13
+# - schemapi version: 0.3.0.dev0+ee22edf
+# - date:    2017-08-09 12:14:26
 
 import pytest
 
@@ -17,7 +17,8 @@ def test_undefined_singleton():
 def generate_test_cases():
     """yield tuples of (trait, failcases, passcases)"""
     # Anys
-    yield (jst.JSONAny(), [], [1, "hello", {'a':2}, [1, 2, 3], None, undefined])
+    yield (jst.JSONAny(), [],
+           [1, "hello", {'a': 2}, [1, 2, 3], None, undefined])
 
     # Nulls
     yield (jst.JSONNull(), [0, "None"], [None, undefined])
@@ -69,7 +70,7 @@ def generate_test_cases():
     yield (jst.JSONEnum([1, "2", None], allow_undefined=False), [undefined], [])
 
     # Instances
-    yield (jst.JSONInstance(dict), [{1}, (1,), [1]], [{1:2}, undefined])
+    yield (jst.JSONInstance(dict), [{1}, (1,), [1]], [{1: 2}, undefined])
     yield (jst.JSONInstance(dict, allow_undefined=False), [undefined], [])
 
     # Unions and other collections
@@ -92,11 +93,12 @@ def test_traits(trait, failcases, passcases):
         trait._validate(obj, passcase)
 
     for failcase in failcases:
-        with pytest.raises(T.TraitError) as err:
+        with pytest.raises(T.TraitError):
             trait._validate(obj, failcase)
 
 
 def test_hastraits_defaults():
+
     class Foo(jst.JSONHasTraits):
         _additional_traits = [T.Integer()]
         name = T.Unicode()
@@ -105,14 +107,15 @@ def test_hastraits_defaults():
     f.set_trait('year', 2000)
     assert set(f.trait_names()) == {'name', 'age', 'year'}
 
-    with pytest.raises(T.TraitError) as err:
+    with pytest.raises(T.TraitError):
         f.set_trait('foo', 'abc')
 
-    with pytest.raises(T.TraitError) as err:
+    with pytest.raises(T.TraitError):
         f.set_trait('age', 'blah')
 
 
 def test_hastraits_required():
+
     class Foo(jst.JSONHasTraits):
         _required_traits = ['name']
         name = jst.JSONString()
@@ -122,7 +125,7 @@ def test_hastraits_required():
     f2 = Foo(age=32)
 
     # contains all required pieces
-    D = f1.to_dict()
+    f1.to_dict()
 
     with pytest.raises(T.TraitError) as err:
         f2.to_dict()
@@ -130,15 +133,17 @@ def test_hastraits_required():
 
 
 def test_no_defaults():
+
     class Foo(jst.JSONHasTraits):
         _additional_traits = False
         name = T.Unicode()
 
-    with pytest.raises(T.TraitError) as err:
-        f = Foo(name="Sarah", year=2000)
+    with pytest.raises(T.TraitError):
+        Foo(name="Sarah", year=2000)
 
 
 def test_AnyOfObject():
+
     class Foo(jst.JSONHasTraits):
         intval = T.Integer()
         flag = T.Bool()
@@ -155,13 +160,13 @@ def test_AnyOfObject():
     FooBar(intval=5, flag=True)
 
     with pytest.raises(T.TraitError):
-        h = FooBar(strval=666, flag=False)
+        FooBar(strval=666, flag=False)
     with pytest.raises(T.TraitError):
-        h = FooBar(strval='hello', flag='bad arg')
+        FooBar(strval='hello', flag='bad arg')
     with pytest.raises(T.TraitError):
-        h = FooBar(intval='bad arg', flag=False)
+        FooBar(intval='bad arg', flag=False)
     with pytest.raises(T.TraitError):
-        h = FooBar(intval=42, flag='bad arg')
+        FooBar(intval=42, flag='bad arg')
 
     # Test from_dict
     FooBar.from_dict({'strval': 'hello', 'flag': True})
@@ -198,6 +203,7 @@ def test_to_from_dict_with_defaults():
 
 
 def test_to_dict_explicit_null():
+
     class MyClass(jst.JSONHasTraits):
         bar = jst.JSONString(allow_none=True, allow_undefined=True)
 
@@ -207,31 +213,38 @@ def test_to_dict_explicit_null():
 
 
 def test_defaults():
+
     class Foo(jst.JSONHasTraits):
         arr = jst.JSONArray(jst.JSONString())
         val = jst.JSONInstance(dict)
+
     assert Foo().to_dict() == {}
 
 
 def test_skip():
+
     class Foo(jst.JSONHasTraits):
         _skip_on_export = ['baz']
         bar = jst.JSONNumber()
         baz = jst.JSONNumber()
+
     f = Foo(bar=1, baz=2)
     assert f.to_dict() == {'bar': 1}
 
 
 def test_finalize():
+
     class Foo(jst.JSONHasTraits):
         bar = jst.JSONNumber()
         bar_times_2 = jst.JSONNumber()
         L = jst.JSONArray(jst.JSONString())
+
         def _finalize(self):
             self.bar_times_2 = 2 * self.bar
             super(Foo, self)._finalize()
+
     f = Foo(bar=4, L=['a', 'b', 'c'])
-    assert f.to_dict() == {'bar': 4, 'bar_times_2': 8, 'L':['a', 'b', 'c']}
+    assert f.to_dict() == {'bar': 4, 'bar_times_2': 8, 'L': ['a', 'b', 'c']}
 
 
 def test_contains():
@@ -257,12 +270,34 @@ def test_to_python():
         e = jst.JSONArray(jst.JSONInstance(Foo))
 
     D = {'c': [1, 2, 3], 'd': {'a': 5, 'b': 'blah'},
-         'e':[{'a': 3, 'b': 'foo'}, {'a': 4, 'b': 'bar'}]}
+         'e': [{'a': 3, 'b': 'foo'}, {'a': 4, 'b': 'bar'}]}
     obj = Bar.from_dict(D)
     obj2 = eval(obj.to_python())
     assert obj2.to_dict() == obj.to_dict() == D
 
     # Make sure there is an error if required traits are missing
     foo = Foo(a=4)
-    with pytest.raises(T.TraitError) as err:
+    with pytest.raises(T.TraitError):
         foo.to_python()
+
+
+def test_metadata():
+    class Foo(jst.JSONHasTraits):
+        bar = jst.JSONInteger()
+
+    dct = {'$schema': 'http://myschema.com/schema.json',
+           '$id': '#/my/context/',
+           'bar': 4}
+    metadata = {key:val for key, val in dct.items()
+                if key.startswith('$')}
+    original_metadata = Foo._metadata.copy()
+
+    # Make sure metadata is properly added
+    f = Foo.from_dict(dct)
+    assert f._metadata == metadata
+
+    # Make sure metadata is properly exported
+    assert f.to_dict() == dct
+
+    # Make sure above doesn't change Foo's internal class dict
+    assert Foo._metadata == original_metadata
diff --git a/altair/tests/test_api.py b/altair/tests/test_api.py
index 04a5b69c..329568f8 100644
--- a/altair/tests/test_api.py
+++ b/altair/tests/test_api.py
@@ -567,3 +567,15 @@ def test_max_rows():
     d = chart.to_dict()
     assert isinstance(d, dict)
     assert 'max_rows' not in d
+
+
+def test_schema_url():
+    chart = make_chart()
+
+    # Make sure that $schema is added to the output
+    dct = chart.to_dict()
+    assert '$schema' in dct
+    assert dct['$schema'] == schema.vegalite_schema_url
+
+    # Make sure that $schema
+    chart = Chart.from_dict(dct)