Repository: bokeh/bokeh

Issue:
Test bokehjs/examples and run them on travis ci
They are constantly broken due to not being continuously tested.


Discussion:

Resolving patch:
diff --git a/examples/plotting/file/airports_map.py b/examples/plotting/file/airports_map.py
--- a/examples/plotting/file/airports_map.py
+++ b/examples/plotting/file/airports_map.py
@@ -31,7 +31,7 @@ def plot(tile_source, filename):
 
 # create a tile source
 tile_options = {}
-tile_options['url'] = 'http://otile2.mqcdn.com/tiles/1.0.0/sat/{Z}/{X}/{Y}.png'
+# TODO tile_options['url'] = 'http://otile2.mqcdn.com/tiles/1.0.0/sat/{Z}/{X}/{Y}.png'
 mq_tile_source = WMTSTileSource(**tile_options)
 
 carto = plot(CARTODBPOSITRON, 'airports_map_cartodb.html')


Test patch:
diff --git a/tests/examples/examples.yaml b/examples/examples.yaml
similarity index 100%
rename from tests/examples/examples.yaml
rename to examples/examples.yaml
diff --git a/tests/examples/collect_examples.py b/tests/examples/collect_examples.py
--- a/tests/examples/collect_examples.py
+++ b/tests/examples/collect_examples.py
@@ -15,31 +15,31 @@
 import logging
 logging.getLogger('requests.packages.urllib3.connectionpool').setLevel(logging.INFO)
 
-base_dir = dirname(__file__)
-example_dir = abspath(join(base_dir, pardir, pardir, 'examples'))
-
 class Flags(object):
-    file     = 1 << 0
-    server   = 1 << 1
-    notebook = 1 << 2
-    slow     = 1 << 3  # example needs a lot of time to run (> 30 s) (e.g. choropleth.py)
-    skip     = 1 << 4  # don't run example at all (e.g. notebooks are completely broken)
-    no_js    = 1 << 5  # skip bokehjs and thus image diff (e.g. google maps key issue)
-    no_diff  = 1 << 6  # skip only image diff (e.g. inherent randomness as in jitter)
+    js       = 1 << 0
+    file     = 1 << 1
+    server   = 1 << 2
+    notebook = 1 << 3
+    slow     = 1 << 4  # example needs a lot of time to run (> 30 s) (e.g. choropleth.py)
+    skip     = 1 << 5  # don't run example at all (e.g. notebooks are completely broken)
+    no_js    = 1 << 6  # skip bokehjs and thus image diff (e.g. google maps key issue)
+    no_diff  = 1 << 7  # skip only image diff (e.g. inherent randomness as in jitter)
 
 
 class Example(object):
 
-    def __init__(self, path, flags):
+    def __init__(self, path, flags, examples_dir):
         self.path = path
         self.flags = flags
+        self.examples_dir = examples_dir
         self._diff_ref = None
         self._upload = False
         self.pixels = 0
         self._has_ref = False
 
     def __str__(self):
-        flags = ["file"     if self.is_file     else "",
+        flags = ["js"       if self.is_js       else "",
+                 "file"     if self.is_file     else "",
                  "server"   if self.is_server   else "",
                  "notebook" if self.is_notebook else "",
                  "slow"     if self.is_slow     else "",
@@ -52,11 +52,19 @@ def __str__(self):
 
     @property
     def name(self):
-        return basename(self.path._no_ext)
+        return basename(self.path_no_ext)
+
+    @property
+    def base_dir(self):
+        return dirname(self.path)
+
+    @property
+    def imgs_dir(self):
+        return join(dirname(self.path), ".tests")
 
     @property
     def relpath(self):
-        return relpath(self.path, example_dir)
+        return relpath(self.path, self.examples_dir)
 
     @property
     def path_no_ext(self):
@@ -66,6 +74,10 @@ def path_no_ext(self):
     def relpath_no_ext(self):
         return splitext(self.relpath)[0]
 
+    @property
+    def is_js(self):
+        return self.flags & Flags.js
+
     @property
     def is_file(self):
         return self.flags & Flags.file
@@ -108,15 +120,15 @@ def diff_path_or_url(self):
 
     @property
     def img_path(self):
-        return "%s-%s-%s.png" % (self.path_no_ext, __version__, job_id)
+        return join(self.imgs_dir, "%s-%s-%s.png" % (self.name, __version__, job_id))
 
     @property
     def ref_path(self):
-        return "%s-%s-%s.png" % (self.path_no_ext, self._diff_ref, job_id)
+        return join(self.imgs_dir, "%s-%s-%s.png" % (self.name, self._diff_ref, job_id))
 
     @property
     def diff_path(self):
-        return "%s-%s-%s-diff-%s.png" % (self.path_no_ext, __version__, self._diff_ref, job_id)
+        return join(self.imgs_dir, "%s-%s-%s-diff-%s.png" % (self.name, __version__, self._diff_ref, job_id))
 
     @property
     def img_url(self):
@@ -171,53 +183,58 @@ def images_differ(self):
     def dimensions_differ(self):
         return self.pixels == -1
 
-def add_examples(list_of_examples, path, example_type=None, slow=None, skip=None, no_js=None, no_diff=None):
-    example_path = join(example_dir, path)
-
-    def get_flags(f):
-        if example_type is not None:
-            return example_type
-        else:
-            return Flags.file
+def add_examples(list_of_examples, path, examples_dir, example_type=None, slow=None, skip=None, no_js=None, no_diff=None):
+    if path == '*':
+        example_path = examples_dir
+    else:
+        example_path = join(examples_dir, path)
 
-    for f in sorted(os.listdir(example_path)):
+    for name in sorted(os.listdir(example_path)):
         flags = 0
+        orig_name = name
 
-        if f.startswith(('_', '.')):
+        if name.startswith(('_', '.')):
             continue
-        elif f.endswith(".py"):
-            flags |= get_flags(f)
-        elif f.endswith(".ipynb"):
+        elif name.endswith(".py"):
+            flags |= example_type if example_type else Flags.file
+        elif name.endswith(".ipynb"):
             flags |= Flags.notebook
-        elif os.path.isdir(os.path.join(example_path, f)):
-            f = os.path.join(f, f + ".py")
-            if not os.path.exists(f):
-                continue
-            flags |= get_flags(f)
+        elif os.path.isdir(join(example_path, name)):
+            full_path = join(example_path, name, name + ".html")
+            if os.path.exists(full_path):
+                name = join(name, name + ".html")
+                flags |= example_type if example_type else Flags.js
+            else:
+                full_path = join(example_path, name, name + ".py")
+                if not os.path.exists(full_path):
+                    continue
+                name = join(name, name + ".py")
+                flags |= example_type if example_type else Flags.file
         else:
             continue
 
-        if slow is not None and f in slow:
+        if slow is not None and orig_name in slow:
             flags |= Flags.slow
 
-        if skip is not None and (skip == 'all' or f in skip):
+        if skip is not None and (skip == 'all' or orig_name in skip):
             flags |= Flags.skip
 
-        if no_js is not None and (no_js == 'all' or f in no_js):
+        if no_js is not None and (no_js == 'all' or orig_name in no_js):
             flags |= Flags.no_js
 
-        if no_diff is not None and (no_diff == 'all' or f in no_diff):
+        if no_diff is not None and (no_diff == 'all' or orig_name in no_diff):
             flags |= Flags.no_diff
 
-        list_of_examples.append(Example(join(example_path, f), flags))
+        list_of_examples.append(Example(join(example_path, name), flags, examples_dir))
 
     return list_of_examples
 
 
-def collect_examples():
+def collect_examples(config_path):
+    examples_dir = dirname(config_path)
     list_of_examples = []
 
-    with open(join(dirname(__file__), "examples.yaml"), "r") as f:
+    with open(config_path, "r") as f:
         examples = yaml.load(f.read())
 
     for example in examples:
@@ -232,8 +249,8 @@ def collect_examples():
         no_js_status = example.get("no_js")
         no_diff_status = example.get("no_diff")
 
-        list_of_examples = add_examples(list_of_examples, path, example_type=example_type,
-            slow=slow_status, skip=skip_status, no_js=no_js_status, no_diff=no_diff_status)
+        list_of_examples = add_examples(list_of_examples, path, examples_dir,
+            example_type=example_type, slow=slow_status, skip=skip_status, no_js=no_js_status, no_diff=no_diff_status)
 
 
     return list_of_examples
diff --git a/tests/examples/examples_report_plugin.py b/tests/examples/examples_report_plugin.py
--- a/tests/examples/examples_report_plugin.py
+++ b/tests/examples/examples_report_plugin.py
@@ -8,7 +8,7 @@
 import pytest
 import jinja2
 
-from os.path import join, dirname, isfile, relpath
+from os.path import abspath, pardir, join, dirname, isfile, relpath
 from py.xml import html
 
 from tests.plugins.constants import __version__
@@ -43,7 +43,11 @@ def pytest_addoption(parser):
 def get_all_examples(config):
     global _examples
     if _examples is None:
-        _examples = collect_examples()
+        base_dir = abspath(join(dirname(__file__), pardir, pardir))
+
+        _examples = []
+        _examples.extend(collect_examples(join(base_dir, "examples", "examples.yaml")))
+        _examples.extend(collect_examples(join(base_dir, "bokehjs", "examples", "examples.yaml")))
 
         for example in _examples:
             example._diff_ref = config.option.diff_ref
@@ -59,6 +63,9 @@ def pytest_generate_tests(metafunc):
     if 'example' in metafunc.fixturenames:
         examples = get_all_examples(metafunc.config)
 
+        if 'js_example' in metafunc.fixturenames:
+            js_examples = [ e for e in examples if e.is_js ]
+            metafunc.parametrize('js_example,example', zip([ e.path for e in js_examples ], js_examples))
         if 'file_example' in metafunc.fixturenames:
             file_examples = [ e for e in examples if e.is_file ]
             metafunc.parametrize('file_example,example', zip([ e.path for e in file_examples ], file_examples))
diff --git a/tests/examples/test_examples.py b/tests/examples/test_examples.py
--- a/tests/examples/test_examples.py
+++ b/tests/examples/test_examples.py
@@ -12,18 +12,29 @@
 from tests.plugins.phantomjs_screenshot import get_phantomjs_screenshot
 from tests.plugins.image_diff import image_diff
 
-from .collect_examples import example_dir
 from .utils import deal_with_output_cells
 
+@pytest.mark.examples
+def test_js_examples(js_example, example, report):
+    if example.is_skip:
+        pytest.skip("skipping %s" % example.relpath)
+
+    if example.no_js:
+        if not pytest.config.option.no_js:
+            warn("skipping bokehjs for %s" % example.relpath)
+    else:
+        _assert_snapshot(example, "file://%s" % example.path, 'js')
+
+        if example.no_diff:
+            warn("skipping image diff for %s" % example.relpath)
+        else:
+            _get_pdiff(example)
 
 @pytest.mark.examples
 def test_file_examples(file_example, example, report):
     if example.is_skip:
         pytest.skip("skipping %s" % example.relpath)
 
-    html_file = "%s.html" % example.path_no_ext
-    url = 'file://' + html_file
-
     (status, duration, out, err) = _run_example(example)
     info("Example run in %s" % white("%.3fs" % duration))
 
@@ -44,7 +55,7 @@ def test_file_examples(file_example, example, report):
         if not pytest.config.option.no_js:
             warn("skipping bokehjs for %s" % example.relpath)
     else:
-        _assert_snapshot(example, url, 'file')
+        _assert_snapshot(example, "file://%s.html" % example.path_no_ext, 'file')
 
         if example.no_diff:
             warn("skipping image diff for %s" % example.relpath)
@@ -143,15 +154,9 @@ def _print_phantomjs_output(result):
 
         info(msg, label="JS")
 
-    # Process resources
     for resource in resources:
-        url = resource['url']
-        if url.endswith(".png"):
-            ok("%s: %s (%s)" % (url, yellow(resource['status']), resource['statusText']))
-        else:
-            fail("Resource error:: %s: %s (%s)" % (url, red(resource['status']), resource['statusText']), label="JS")
+        fail(resource['errorString'], label="JS")
 
-    # You can have a successful test, and still have errors reported, so not failing here.
     for error in errors:
         fail(error['msg'], label="JS")
         for item in error['trace']:
@@ -181,6 +186,7 @@ def _assert_snapshot(example, url, example_type):
     resources = result['resources']
 
     no_errors = len(errors) == 0
+    no_resources = len(resources) == 0
 
     if timeout:
         warn("%s: %s" % (red("TIMEOUT: "), "bokehjs did not finish in %s ms" % wait))
@@ -188,8 +194,9 @@ def _assert_snapshot(example, url, example_type):
     if pytest.config.option.verbose:
         _print_phantomjs_output(result)
 
-    assert success, "Example failed to load"
-    assert no_errors, "Example failed with %d errors" % len(errors)
+    assert success, "%s failed to load" % example.relpath
+    assert no_resources, "%s failed with %d missing resources" % (example.relpath, len(resources))
+    assert no_errors, "%s failed with %d errors" % (example.relpath, len(errors))
 
 
 def _run_example(example):
diff --git a/tests/plugins/phantomjs_screenshot.js b/tests/plugins/phantomjs_screenshot.js
--- a/tests/plugins/phantomjs_screenshot.js
+++ b/tests/plugins/phantomjs_screenshot.js
@@ -50,28 +50,14 @@ page.onError = function(msg, trace) {
   });
 };
 
-page.onConsoleMessage = function(msg, line, source) {
-  if (localTimer != null) resetLocalTimer();
-  messages.push({msg: msg, line: line, source: source});
+page.onResourceError = function(response) {
+  resetLocalTimer();
+  resources.push(response);
 };
 
-page.onResourceReceived = function(response) {
+page.onConsoleMessage = function(msg, line, source) {
   if (localTimer != null) resetLocalTimer();
-  if (response.stage === 'end') {
-    var status = response.status;
-
-    if (response.url.slice(0, 7) === "file://") {
-      var path = response.url.slice(7);
-
-      if (!fs.exists(path)) {
-        response.status = 404;
-        response.statusText = "NOT FOUND";
-        resources.push(response);
-      }
-    } else if (status && status >= 400) {
-      resources.push(response);
-    }
-  }
+  messages.push({msg: msg, line: line, source: source});
 };
 
 page.viewportSize = { width: width, height: height };