Repository: bokeh/bokeh

Issue:
Re-enable Saucelabs tests
All saucelabs jobs on all builds are failing with errors similar to this:
```
DEBUG:selenium.webdriver.remote.remote_connection:GET http://bokehplots:f7047da0-5e51-4f77-9761-cb741be552be@162.222.75.33:80/wd/hub/session/9d037f3c02894d4cb2d24fc57a1bfa33/url {"sessionId": "9d037f3c02894d4cb2d24fc57a1bfa33"}
DEBUG:selenium.webdriver.remote.remote_connection:Finished Request
DEBUG:selenium.webdriver.remote.remote_connection:GET http://bokehplots:f7047da0-5e51-4f77-9761-cb741be552be@162.222.75.33:80/wd/hub/session/9d037f3c02894d4cb2d24fc57a1bfa33/screenshot {"sessionId": "9d037f3c02894d4cb2d24fc57a1bfa33"}
DEBUG:selenium.webdriver.remote.remote_connection:Finished Request
INTERNALERROR> Traceback (most recent call last):
INTERNALERROR>   File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/main.py", line 98, in wrap_session
INTERNALERROR>     session.exitstatus = doit(config, session) or 0
INTERNALERROR>   File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/main.py", line 133, in _main
INTERNALERROR>     config.hook.pytest_runtestloop(session=session)
INTERNALERROR>   File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 745, in __call__
INTERNALERROR>     return self._hookexec(self, self._nonwrappers + self._wrappers, kwargs)
INTERNALERROR>   File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 339, in _hookexec
INTERNALERROR>     return self._inner_hookexec(hook, methods, kwargs)
INTERNALERROR>   File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 334, in <lambda>
INTERNALERROR>     _MultiCall(methods, kwargs, hook.spec_opts).execute()
INTERNALERROR>   File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 614, in execute
INTERNALERROR>     res = hook_impl.function(*args)
INTERNALERROR>   File "<remote exec>", line 58, in pytest_runtestloop
INTERNALERROR>   File "<remote exec>", line 74, in run_tests
INTERNALERROR>   File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 745, in __call__
INTERNALERROR>     return self._hookexec(self, self._nonwrappers + self._wrappers, kwargs)
INTERNALERROR>   File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 339, in _hookexec
INTERNALERROR>     return self._inner_hookexec(hook, methods, kwargs)
INTERNALERROR>   File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 334, in <lambda>
INTERNALERROR>     _MultiCall(methods, kwargs, hook.spec_opts).execute()
INTERNALERROR>   File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 613, in execute
INTERNALERROR>     return _wrapped_call(hook_impl.function(*args), self.execute)
INTERNALERROR>   File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 254, in _wrapped_call
INTERNALERROR>     return call_outcome.get_result()
INTERNALERROR>   File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 279, in get_result
INTERNALERROR>     raise ex[1].with_traceback(ex[2])
INTERNALERROR>   File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 265, in __init__
INTERNALERROR>     self.result = func()
INTERNALERROR>   File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 614, in execute
INTERNALERROR>     res = hook_impl.function(*args)
INTERNALERROR>   File "/home/travis/miniconda/lib/python3.4/site-packages/pytest_rerunfailures-1.0.1-py3.4.egg/pytest_rerunfailures.py", line 65, in pytest_runtest_protocol
INTERNALERROR>     reports = runtestprotocol(item, nextitem=nextitem, log=False)
INTERNALERROR>   File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/runner.py", line 79, in runtestprotocol
INTERNALERROR>     reports.append(call_and_report(item, "call", log))
INTERNALERROR>   File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/runner.py", line 135, in call_and_report
INTERNALERROR>     report = hook.pytest_runtest_makereport(item=item, call=call)
INTERNALERROR>   File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 745, in __call__
INTERNALERROR>     return self._hookexec(self, self._nonwrappers + self._wrappers, kwargs)
INTERNALERROR>   File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 339, in _hookexec
INTERNALERROR>     return self._inner_hookexec(hook, methods, kwargs)
INTERNALERROR>   File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 334, in <lambda>
INTERNALERROR>     _MultiCall(methods, kwargs, hook.spec_opts).execute()
INTERNALERROR>   File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 613, in execute
INTERNALERROR>     return _wrapped_call(hook_impl.function(*args), self.execute)
INTERNALERROR>   File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 250, in _wrapped_call
INTERNALERROR>     wrap_controller.send(call_outcome)
INTERNALERROR>   File "/home/travis/build/bokeh/bokeh/tests/integration/integration_tests_plugin.py", line 139, in pytest_runtest_makereport
INTERNALERROR>     test = pytest_html.extras.image(screenshot.get_current_as_base64(), '')
INTERNALERROR>   File "/home/travis/build/bokeh/bokeh/tests/integration/screenshot.py", line 58, in get_current_as_base64
INTERNALERROR>     return self.get_screenshot_as_b64(self.current_screenshot_path)
INTERNALERROR>   File "/home/travis/build/bokeh/bokeh/tests/integration/screenshot.py", line 37, in get_screenshot_as_b64
INTERNALERROR>     with open(path, 'rb') as f:
INTERNALERROR> FileNotFoundError: [Errno 2] No such file or directory: '/home/travis/build/bokeh/bokeh/tests/integration/annotations/screenshots/current__test_label__test_label.png'
--- Logging error ---
Traceback (most recent call last):
  File "/home/travis/miniconda/lib/python3.4/logging/__init__.py", line 980, in emit
    stream.write(msg)
  File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/capture.py", line 244, in write
    self.buffer.write(obj)
ValueError: I/O operation on closed file
Call stack:
  File "<string>", line 1, in <module>
  File "<string>", line 7, in <module>
  File "/home/travis/miniconda/lib/python3.4/site-packages/execnet/gateway_base.py", line 1469, in serve
    SlaveGateway(io=io, id=id, _startcount=2).serve()
  File "/home/travis/miniconda/lib/python3.4/site-packages/execnet/gateway_base.py", line 1014, in serve
    self._execpool.integrate_as_primary_thread()
  File "/home/travis/miniconda/lib/python3.4/site-packages/execnet/gateway_base.py", line 246, in integrate_as_primary_thread
    self._perform_spawn(reply)
  File "/home/travis/miniconda/lib/python3.4/site-packages/execnet/gateway_base.py", line 264, in _perform_spawn
    reply.run()
  File "/home/travis/miniconda/lib/python3.4/site-packages/execnet/gateway_base.py", line 200, in run
    self._result = func(*args, **kwargs)
  File "/home/travis/miniconda/lib/python3.4/site-packages/execnet/gateway_base.py", line 1039, in executetask
    do_exec(co, loc) # noqa
  File "<string>", line 1, in do_exec
  File "<remote exec>", line 154, in <module>
  File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 745, in __call__
    return self._hookexec(self, self._nonwrappers + self._wrappers, kwargs)
  File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 339, in _hookexec
    return self._inner_hookexec(hook, methods, kwargs)
  File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 334, in <lambda>
    _MultiCall(methods, kwargs, hook.spec_opts).execute()
  File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 614, in execute
    res = hook_impl.function(*args)
  File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/main.py", line 127, in pytest_cmdline_main
    return wrap_session(config, _main)
  File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/main.py", line 122, in wrap_session
    exitstatus=session.exitstatus)
  File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 745, in __call__
    return self._hookexec(self, self._nonwrappers + self._wrappers, kwargs)
  File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 339, in _hookexec
    return self._inner_hookexec(hook, methods, kwargs)
  File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 334, in <lambda>
    _MultiCall(methods, kwargs, hook.spec_opts).execute()
  File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 614, in execute
    res = hook_impl.function(*args)
  File "<remote exec>", line 38, in pytest_sessionfinish
  File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/vendored_packages/pluggy.py", line 614, in execute
    res = hook_impl.function(*args)
  File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/runner.py", line 56, in pytest_sessionfinish
    session._setupstate.teardown_all()
  File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/runner.py", line 425, in teardown_all
    self._pop_and_teardown()
  File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/runner.py", line 398, in _pop_and_teardown
    self._teardown_with_finalization(colitem)
  File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/runner.py", line 416, in _teardown_with_finalization
    self._callfinalizers(colitem)
  File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/runner.py", line 406, in _callfinalizers
    fin()
  File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/fixtures.py", line 755, in finish
    func()
  File "/home/travis/miniconda/lib/python3.4/site-packages/_pytest/fixtures.py", line 713, in teardown
    next(it)
  File "/home/travis/miniconda/lib/python3.4/site-packages/pytest_selenium-1.6.0-py3.6.egg/pytest_selenium/pytest_selenium.py", line 120, in driver
    driver.quit()
  File "/home/travis/miniconda/lib/python3.4/site-packages/selenium/webdriver/remote/webdriver.py", line 518, in quit
    self.execute(Command.QUIT)
  File "/home/travis/miniconda/lib/python3.4/site-packages/selenium/webdriver/remote/webdriver.py", line 234, in execute
    response = self.command_executor.execute(driver_command, params)
  File "/home/travis/miniconda/lib/python3.4/site-packages/selenium/webdriver/remote/remote_connection.py", line 401, in execute
    return self._request(command_info[0], url, body=data)
  File "/home/travis/miniconda/lib/python3.4/site-packages/selenium/webdriver/remote/remote_connection.py", line 415, in _request
    LOGGER.debug('%s %s %s' % (method, url, body))
Message: 'DELETE http://bokehplots:f7047da0-5e51-4f77-9761-cb741be552be@162.222.75.33:80/wd/hub/session/9d037f3c02894d4cb2d24fc57a1bfa33 {"sessionId": "9d037f3c02894d4cb2d24fc57a1bfa33"}'
Arguments: ()
```


Discussion:
Verified FF version (45) has not changed on Saucelabs

Resolving patch:
diff --git a/bokeh/io.py b/bokeh/io.py
--- a/bokeh/io.py
+++ b/bokeh/io.py
@@ -442,10 +442,10 @@ def _get_save_args(state, filename, resources, title):
 def _save_helper(obj, filename, resources, title, validate):
     remove_after = False
     if isinstance(obj, LayoutDOM):
-        doc = obj.document
-        if doc is None:
-            doc = Document().add_root(obj)
+        if obj.document is None:
+            Document().add_root(obj)
             remove_after = True
+        doc = obj.document
     elif isinstance(obj, Document):
         doc = obj
     else:
@@ -454,9 +454,9 @@ def _save_helper(obj, filename, resources, title, validate):
     if validate:
         doc.validate()
 
-    html = standalone_html_page_for_models(obj, resources, title)
+    html = standalone_html_page_for_models(doc, resources, title)
 
-    with io.open(filename, "w", encoding="utf-8") as f:
+    with io.open(filename, mode="w", encoding="utf-8") as f:
         f.write(decode_utf8(html))
 
     if remove_after:


Test patch:
diff --git a/bokeh/tests/test_io.py b/bokeh/tests/test_io.py
--- a/bokeh/tests/test_io.py
+++ b/bokeh/tests/test_io.py
@@ -152,7 +152,36 @@ def test_missing_title(self, mock_warn):
         self.assertEqual(mock_warn.call_args[1], {})
 
 class Test_SaveHelper(DefaultStateTester):
-    pass
+
+    @patch('io.open')
+    @patch('bokeh.io.standalone_html_page_for_models')
+    def test_obj_arg_is_document(self, mock_standalone_html_page, mock_io_open):
+        obj = Document()
+        filename, resources, title = io._get_save_args(io._state, "filename", "resources", "title")
+        io._save_helper(obj, filename, resources, title, True)
+
+        self._check_func_called(mock_standalone_html_page,
+                                (obj, resources, title),
+                                {})
+        self._check_func_called(mock_io_open,
+                                (filename,),
+                                {"mode":"w", "encoding":"utf-8"})
+
+    @patch('io.open')
+    @patch('bokeh.io.standalone_html_page_for_models')
+    def test_obj_arg_is_layoutdom(self, mock_standalone_html_page, mock_io_open):
+        obj = Plot()
+        doc = Document()
+        doc.add_root(obj)
+        filename, resources, title = io._get_save_args(io._state, "filename", "resources", "title")
+        io._save_helper(obj, filename, resources, title, True)
+
+        self._check_func_called(mock_standalone_html_page,
+                                (doc, resources, title),
+                                {})
+        self._check_func_called(mock_io_open,
+                                (filename,),
+                                {"mode":"w", "encoding":"utf-8"})
 
 class TestPush(DefaultStateTester):
 
diff --git a/tests/integration/annotations/test_arrow.py b/tests/integration/annotations/test_arrow.py
--- a/tests/integration/annotations/test_arrow.py
+++ b/tests/integration/annotations/test_arrow.py
@@ -44,4 +44,4 @@ def test_arrow(output_file_url, selenium, screenshot):
     assert has_no_console_errors(selenium)
 
     # Take screenshot
-    assert screenshot.is_valid()
+    screenshot.assert_is_valid()
diff --git a/tests/integration/annotations/test_color_bar.py b/tests/integration/annotations/test_color_bar.py
--- a/tests/integration/annotations/test_color_bar.py
+++ b/tests/integration/annotations/test_color_bar.py
@@ -60,7 +60,7 @@ def test_color_bar_placement_and_render(output_file_url, selenium, screenshot):
     assert has_no_console_errors(selenium)
 
     # Take screenshot
-    assert screenshot.is_valid()
+    screenshot.assert_is_valid()
 
 @pytest.mark.screenshot
 def test_color_bar_with_scale_alpha(output_file_url, selenium, screenshot):
@@ -79,4 +79,4 @@ def test_color_bar_with_scale_alpha(output_file_url, selenium, screenshot):
     assert has_no_console_errors(selenium)
 
     # Take screenshot
-    assert screenshot.is_valid()
+    screenshot.assert_is_valid()
diff --git a/tests/integration/annotations/test_label.py b/tests/integration/annotations/test_label.py
--- a/tests/integration/annotations/test_label.py
+++ b/tests/integration/annotations/test_label.py
@@ -71,4 +71,4 @@ def test_label(output_file_url, selenium, screenshot):
     assert has_no_console_errors(selenium)
 
     # Take screenshot
-    assert screenshot.is_valid()
+    screenshot.assert_is_valid()
diff --git a/tests/integration/annotations/test_label_set.py b/tests/integration/annotations/test_label_set.py
--- a/tests/integration/annotations/test_label_set.py
+++ b/tests/integration/annotations/test_label_set.py
@@ -49,4 +49,4 @@ def test_label_set(output_file_url, selenium, screenshot):
     selenium.get(output_file_url)
 
     # Take screenshot
-    assert screenshot.is_valid()
+    screenshot.assert_is_valid()
diff --git a/tests/integration/annotations/test_legend.py b/tests/integration/annotations/test_legend.py
--- a/tests/integration/annotations/test_legend.py
+++ b/tests/integration/annotations/test_legend.py
@@ -37,4 +37,4 @@ def test_legend_powered_by_source(output_file_url, selenium, screenshot):
     assert has_no_console_errors(selenium)
 
     # Take screenshot
-    assert screenshot.is_valid()
+    screenshot.assert_is_valid()
diff --git a/tests/integration/annotations/test_title.py b/tests/integration/annotations/test_title.py
--- a/tests/integration/annotations/test_title.py
+++ b/tests/integration/annotations/test_title.py
@@ -45,4 +45,4 @@ def make_plot(location, title_align, two_axes=True):
     selenium.get(output_file_url)
 
     # Take screenshot
-    assert screenshot.is_valid()
+    screenshot.assert_is_valid()
diff --git a/tests/integration/integration_tests_plugin.py b/tests/integration/integration_tests_plugin.py
--- a/tests/integration/integration_tests_plugin.py
+++ b/tests/integration/integration_tests_plugin.py
@@ -2,7 +2,7 @@
 import pytest
 
 from bokeh.io import output_file
-from .screenshot import Screenshot
+from .screenshot import Screenshot, ScreenshotMismatchError
 
 
 def pytest_addoption(parser):
@@ -21,7 +21,6 @@ def selenium(selenium):
 
 @pytest.fixture
 def output_file_url(request, file_server):
-
     filename = request.function.__name__ + '.html'
     file_obj = request.fspath.dirpath().join(filename)
     file_path = file_obj.strpath
@@ -106,37 +105,23 @@ def screenshot(request):
 
 
 #
-# Hook into the pytest report to add the screnshot diff
+# Hook into the pytest report to add the screenshot diff
 #
 
+
 @pytest.mark.hookwrapper
 def pytest_runtest_makereport(item, call):
     outcome = yield
-
-    # Only run through this at the end of the test
-    if call.when != 'call':
-        return
-
-    # Don't continue if this isn't a screenshot test
-    if 'screenshot' not in item.fixturenames:
-        return
-
-    # Don't add screenshots if we can't create a screenshot
-    try:
-        screenshot = Screenshot(item=item)
-    except AssertionError:
-        return
-
     report = outcome.get_result()
-    xfail = hasattr(report, 'wasxfail')
-    failure = (report.skipped and xfail) or (report.failed and not xfail)
-    pytest_html = item.config.pluginmanager.getplugin('html')
-
-    # Don't add screenshots if test passed
-    if failure:
-        diff = pytest_html.extras.image(screenshot.get_diff_as_base64(), '')
-        base = pytest_html.extras.image(screenshot.get_base_as_base64(), '')
-        test = pytest_html.extras.image(screenshot.get_current_as_base64(), '')
-        # Override existing extra and logs for screenshot tests (keeps output manageable)
-        report.extra = [test, diff, base]
-        report.longrepr = ''
+
+    # When executing the test failed for some reason
+    if report.when == "call" and report.failed:
+
+        if 'screenshot' in item.fixturenames and isinstance(call.excinfo.value, ScreenshotMismatchError):
+            screenshot = Screenshot(item=item)
+            pytest_html = item.config.pluginmanager.getplugin('html')
+            diff = pytest_html.extras.image(screenshot.get_diff_as_base64(), '')
+            base = pytest_html.extras.image(screenshot.get_base_as_base64(), '')
+            test = pytest_html.extras.image(screenshot.get_current_as_base64(), '')
+            # Override existing extra screenshot attr to add image reports
+            report.extra = [test, diff, base]
diff --git a/tests/integration/interaction/test_hover.py b/tests/integration/interaction/test_hover.py
--- a/tests/integration/interaction/test_hover.py
+++ b/tests/integration/interaction/test_hover.py
@@ -40,4 +40,4 @@ def test_hover_changes_color(output_file_url, selenium, screenshot):
     # Hover over plot and take screenshot
     canvas = selenium.find_element_by_tag_name('canvas')
     hover_at_position(selenium, canvas, WIDTH * 0.33, HEIGHT * 0.5)
-    assert screenshot.is_valid()
+    screenshot.assert_is_valid()
diff --git a/tests/integration/interaction/test_range_min_max.py b/tests/integration/interaction/test_range_min_max.py
--- a/tests/integration/interaction/test_range_min_max.py
+++ b/tests/integration/interaction/test_range_min_max.py
@@ -4,7 +4,6 @@
 from bokeh.models import (
     BoxZoomTool,
     ColumnDataSource,
-    CustomJS,
     DataRange1d,
     PanTool,
     Plot,
@@ -13,44 +12,26 @@
     LinearAxis
 )
 from selenium.webdriver.common.action_chains import ActionChains
-from tests.integration.utils import has_no_console_errors
+from tests.integration.utils import has_no_console_errors, wait_for_canvas_resize
 
 import pytest
 pytestmark = pytest.mark.integration
 
-from .test_sizing_mode import wait_for_canvas_resize
 
-def make_pan_plot_with_callback(xr=None, yr=None):
+def make_plot(xr=None, yr=None):
     if xr is None:
         x_range = Range1d(0, 3, bounds=None)
     else:
         x_range = xr
-    x_callback = CustomJS(args=dict(x_range=x_range), code="""
-        window.get_x_range_start = function() {
-            return x_range.start;
-        }
-        window.get_x_range_end = function() {
-            return x_range.end;
-        }
-    """)
-    x_range.callback = x_callback
 
     if yr is None:
         y_range = Range1d(0, 3, bounds=None)
     else:
         y_range = yr
-    y_callback = CustomJS(args=dict(y_range=y_range), code="""
-        window.get_y_range_start = function() {
-            return y_range.start;
-        }
-        window.get_y_range_end = function() {
-            return y_range.end;
-        }
-    """)
-    y_range.callback = y_callback
 
     source = ColumnDataSource(dict(x=[1, 2], y=[1, 1]))
-    plot = Plot(plot_height=400, plot_width=400, x_range=x_range, y_range=y_range, min_border=0)
+    # explicitly set plot.id so that the plot can be accessed from Bokeh.index in browser
+    plot = Plot(id='plot-id', plot_height=400, plot_width=400, x_range=x_range, y_range=y_range, min_border=0)
     plot.add_glyph(source, Rect(x='x', y='y', width=0.9, height=0.9))
     plot.add_tools(PanTool(), BoxZoomTool())
     plot.add_layout(LinearAxis(), 'below')
@@ -75,75 +56,59 @@ def pan_plot(selenium, pan_x=None, pan_y=None):
     actions.perform()
 
 
-def test_range_with_callback_triggers_alert(output_file_url, selenium):
-    # Simple test to ensure range callbacks are working
-    # Rest of tests in this file depend on range callback.
-
-    plot = make_pan_plot_with_callback()
-    initial_start = plot.x_range.start
-    save(plot)
-    selenium.get(output_file_url)
-    assert has_no_console_errors(selenium)
-
-    # Pan plot and test for new range value
-    pan_plot(selenium, pan_x=100, pan_y=100)
-    new_range_start = float(selenium.execute_script("""alert(window.get_x_range_start())"""))
-    selenium.switch_to_alert().dismiss()
-    assert new_range_start < initial_start
-
-
 def test_x_range_does_not_pan_left_of_x_min(output_file_url, selenium):
     x_range_min = -1
-    plot = make_pan_plot_with_callback(xr=Range1d(0, 3, bounds=(x_range_min, None)))
+    plot = make_plot(xr=Range1d(0, 3, bounds=(x_range_min, None)))
     save(plot)
     selenium.get(output_file_url)
     assert has_no_console_errors(selenium)
 
     # Pan plot and test for new range value
-    pan_plot(selenium, pan_x=200, pan_y=0)
-    new_range_start = float(selenium.execute_script("""alert(window.get_x_range_start())"""))
+    pan_plot(selenium, pan_x=150, pan_y=0)
+    new_range_start = float(selenium.execute_script("""alert(Bokeh.index['plot-id'].model.x_range.start)"""))
     selenium.switch_to_alert().dismiss()
     assert round(new_range_start) == x_range_min
 
 
 def test_x_range_does_not_pan_right_of_x_max(output_file_url, selenium):
     x_range_max = 4
-    plot = make_pan_plot_with_callback(xr=Range1d(0, 3, bounds=(None, x_range_max)))
+    plot = make_plot(xr=Range1d(0, 3, bounds=(None, x_range_max)))
     save(plot)
     selenium.get(output_file_url)
     assert has_no_console_errors(selenium)
 
     # Pan plot and test for new range value
-    pan_plot(selenium, pan_x=-200, pan_y=0)
-    new_range_end = float(selenium.execute_script("""alert(window.get_x_range_end())"""))
+    pan_plot(selenium, pan_x=-150, pan_y=0)
+    new_range_end = float(selenium.execute_script("""alert(Bokeh.index['plot-id'].model.x_range.end)"""))
     selenium.switch_to_alert().dismiss()  # This is not necessary but assists debugging
     assert round(new_range_end) == x_range_max
 
 
 def test_y_range_does_not_pan_below_y_min(output_file_url, selenium):
     y_range_min = -1
-    plot = make_pan_plot_with_callback(yr=Range1d(0, 3, bounds=(y_range_min, None)))
+    plot = make_plot(yr=Range1d(0, 3, bounds=(y_range_min, None)))
     save(plot)
     selenium.get(output_file_url)
     assert has_no_console_errors(selenium)
 
     # Pan plot and test for new range value
     pan_plot(selenium, pan_x=50, pan_y=-150)
-    new_range_start = float(selenium.execute_script("""alert(window.get_y_range_start())"""))
+    new_range_start = float(selenium.execute_script("""alert(Bokeh.index['plot-id'].model.y_range.start)"""))
     selenium.switch_to_alert().dismiss()  # This is not necessary but assists debugging
     assert round(new_range_start) == y_range_min
 
 
+
 def test_y_range_does_not_pan_above_y_max(output_file_url, selenium):
     y_range_max = 4
-    plot = make_pan_plot_with_callback(yr=Range1d(0, 3, bounds=(None, y_range_max)))
+    plot = make_plot(yr=Range1d(0, 3, bounds=(None, y_range_max)))
     save(plot)
     selenium.get(output_file_url)
     assert has_no_console_errors(selenium)
 
     # Pan plot and test for new range value
     pan_plot(selenium, pan_x=50, pan_y=150)
-    new_range_end = float(selenium.execute_script("""alert(window.get_y_range_end())"""))
+    new_range_end = float(selenium.execute_script("""alert(Bokeh.index['plot-id'].model.y_range.end)"""))
     selenium.switch_to_alert().dismiss()  # This is not necessary but assists debugging
     assert round(new_range_end) == y_range_max
 
@@ -154,35 +119,35 @@ def test_y_range_does_not_pan_above_y_max(output_file_url, selenium):
 
 def test_reversed_x_range_does_not_pan_right_of_x_min(output_file_url, selenium):
     x_range_min = -1
-    plot = make_pan_plot_with_callback(xr=Range1d(3, 0, bounds=(x_range_min, None)))
+    plot = make_plot(xr=Range1d(3, 0, bounds=(x_range_min, None)))
     save(plot)
     selenium.get(output_file_url)
     assert has_no_console_errors(selenium)
 
     # Pan plot and test for new range value
-    pan_plot(selenium, pan_x=-200, pan_y=0)
-    new_range_start = float(selenium.execute_script("""alert(window.get_x_range_end())"""))
+    pan_plot(selenium, pan_x=-150, pan_y=0)
+    new_range_start = float(selenium.execute_script("""alert(Bokeh.index['plot-id'].model.x_range.min)"""))
     selenium.switch_to_alert().dismiss()
     assert round(new_range_start) == x_range_min
 
 
 def test_reversed_x_range_does_not_pan_left_of_x_max(output_file_url, selenium):
     x_range_max = 4
-    plot = make_pan_plot_with_callback(xr=Range1d(3, 0, bounds=(None, x_range_max)))
+    plot = make_plot(xr=Range1d(3, 0, bounds=(None, x_range_max)))
     save(plot)
     selenium.get(output_file_url)
     assert has_no_console_errors(selenium)
 
     # Pan plot and test for new range value
-    pan_plot(selenium, pan_x=200, pan_y=0)
-    new_range_end = float(selenium.execute_script("""alert(window.get_x_range_start())"""))
+    pan_plot(selenium, pan_x=150, pan_y=0)
+    new_range_end = float(selenium.execute_script("""alert(Bokeh.index['plot-id'].model.x_range.max)"""))
     selenium.switch_to_alert().dismiss()  # This is not necessary but assists debugging
     assert round(new_range_end) == x_range_max
 
 
 def test_reversed_y_range_does_not_pan_above_y_min(output_file_url, selenium):
     y_range_min = -1
-    plot = make_pan_plot_with_callback(yr=Range1d(3, 0, bounds=(y_range_min, None)))
+    plot = make_plot(yr=Range1d(3, 0, bounds=(y_range_min, None)))
     save(plot)
     selenium.get(output_file_url)
     assert has_no_console_errors(selenium)
@@ -190,21 +155,21 @@ def test_reversed_y_range_does_not_pan_above_y_min(output_file_url, selenium):
     # Pan plot and test for new range value
     pan_plot(selenium, pan_x=50, pan_y=150)
 
-    new_range_start = float(selenium.execute_script("""alert(window.get_y_range_end())"""))
+    new_range_start = float(selenium.execute_script("""alert(Bokeh.index['plot-id'].model.y_range.min)"""))
     selenium.switch_to_alert().dismiss()
     assert round(new_range_start) == y_range_min
 
 
 def test_reversed_y_range_does_not_pan_below_y_max(output_file_url, selenium):
     y_range_max = 4
-    plot = make_pan_plot_with_callback(yr=Range1d(3, 0, bounds=(None, y_range_max)))
+    plot = make_plot(yr=Range1d(3, 0, bounds=(None, y_range_max)))
     save(plot)
     selenium.get(output_file_url)
     assert has_no_console_errors(selenium)
 
     # Pan plot and test for new range value
     pan_plot(selenium, pan_x=50, pan_y=-150)
-    new_range_end = float(selenium.execute_script("""alert(window.get_y_range_start())"""))
+    new_range_end = float(selenium.execute_script("""alert(Bokeh.index['plot-id'].model.y_range.max)"""))
     selenium.switch_to_alert().dismiss()
     assert round(new_range_end) == y_range_max
 
@@ -239,26 +204,26 @@ def _assert_autorange_prevents_panning_but_can_zoom(output_file_url, selenium):
 
     # Now the plot is zoomed in, try a little to the right
     pan_plot(selenium, pan_x=-50, pan_y=0)
-    x_range_start = float(selenium.execute_script("""alert(window.get_x_range_start())"""))
+    x_range_start = float(selenium.execute_script("""alert(Bokeh.index['plot-id'].model.x_range.start)"""))
     selenium.switch_to_alert().dismiss()
     assert x_range_start > 0.5
 
     # Now try panning far to left to check bounds
-    pan_plot(selenium, pan_x=200, pan_y=0)
-    x_range_start = float(selenium.execute_script("""alert(window.get_x_range_start())"""))
+    pan_plot(selenium, pan_x=100, pan_y=0)
+    x_range_start = float(selenium.execute_script("""alert(Bokeh.index['plot-id'].model.x_range.start)"""))
     selenium.switch_to_alert().dismiss()
     assert x_range_start > 0.4
     assert x_range_start < 0.5
 
 
 def test_autorange_prevents_panning_but_can_zoom_in_with_datarange1d(output_file_url, selenium):
-    plot = make_pan_plot_with_callback(xr=DataRange1d(bounds='auto'), yr=DataRange1d(bounds='auto'))
+    plot = make_plot(xr=DataRange1d(bounds='auto'), yr=DataRange1d(bounds='auto'))
     save(plot)
     _assert_autorange_prevents_panning_but_can_zoom(output_file_url, selenium)
 
 
 def test_autorange_prevents_panning_but_can_zoom_in_with_range1d(output_file_url, selenium):
-    plot = make_pan_plot_with_callback(xr=Range1d(0.45, 3, bounds='auto'), yr=DataRange1d(0, 3, bounds='auto'))
+    plot = make_plot(xr=Range1d(0.45, 3, bounds='auto'), yr=DataRange1d(0, 3, bounds='auto'))
     save(plot)
     _assert_autorange_prevents_panning_but_can_zoom(output_file_url, selenium)
 
@@ -282,12 +247,12 @@ def test_autorange_prevents_panning_but_can_zoom_in_with_range1d(output_file_url
 #
 #
 #def test_no_bounds_allows_unlimited_panning_with_datarange1d(output_file_url, selenium):
-#    plot = make_pan_plot_with_callback(xr=DataRange1d(bounds=None), yr=DataRange1d(bounds=None))
+#    plot = make_plot_with_callback(xr=DataRange1d(bounds=None), yr=DataRange1d(bounds=None))
 #    save(plot)
 #    _assert_no_bounds_allows_unlimited_panning(output_file_url, selenium)
 #
 #
 #def test_no_bounds_allows_unlimited_panning_with_range1d(output_file_url, selenium):
-#    plot = make_pan_plot_with_callback(xr=Range1d(0.45, 3, bounds=None), yr=DataRange1d(0, 3, bounds=None))
+#    plot = make_plot_with_callback(xr=Range1d(0.45, 3, bounds=None), yr=DataRange1d(0, 3, bounds=None))
 #    save(plot)
 #    _assert_no_bounds_allows_unlimited_panning(output_file_url, selenium)
diff --git a/tests/integration/plotting/test_plot_fill_properties.py b/tests/integration/plotting/test_plot_fill_properties.py
--- a/tests/integration/plotting/test_plot_fill_properties.py
+++ b/tests/integration/plotting/test_plot_fill_properties.py
@@ -67,4 +67,4 @@ def test_no_border_or_background_fill(output_file_url, selenium, screenshot):
 
     selenium.get(output_file_url)
     assert has_no_console_errors(selenium)
-    assert screenshot.is_valid()
+    screenshot.assert_is_valid()
diff --git a/tests/integration/plotting/test_rect_glyph_with_log_axis.py b/tests/integration/plotting/test_rect_glyph_with_log_axis.py
--- a/tests/integration/plotting/test_rect_glyph_with_log_axis.py
+++ b/tests/integration/plotting/test_rect_glyph_with_log_axis.py
@@ -26,4 +26,4 @@ def test_rect_rendering_with_log_axis(output_file_url, selenium, screenshot):
     save(plot)
     selenium.get(output_file_url)
     assert has_no_console_errors(selenium)
-    assert screenshot.is_valid()
+    screenshot.assert_is_valid()
diff --git a/tests/integration/plotting/test_visible_property.py b/tests/integration/plotting/test_visible_property.py
--- a/tests/integration/plotting/test_visible_property.py
+++ b/tests/integration/plotting/test_visible_property.py
@@ -22,4 +22,4 @@ def test_visible_property_hides_things_correctly(output_file_url, selenium, scre
     save(plot)
     selenium.get(output_file_url)
     assert has_no_console_errors(selenium)
-    assert screenshot.is_valid()
+    screenshot.assert_is_valid()
diff --git a/tests/integration/screenshot.py b/tests/integration/screenshot.py
--- a/tests/integration/screenshot.py
+++ b/tests/integration/screenshot.py
@@ -4,6 +4,12 @@
 from tests.plugins.image_diff import image_diff
 
 
+class ScreenshotMismatchError(AssertionError):
+    """ Custom assertion error for report handling. """
+    def __init__(self, *args, **kwargs):
+        super(AssertionError, self).__init__(*args, **kwargs)
+
+
 class Screenshot(object):
 
     def __init__(self, item=None, request=None, set_new_base=False):
@@ -57,7 +63,7 @@ def get_current_as_base64(self):
         if exists(self.base_screenshot_path):
             return self.get_screenshot_as_b64(self.current_screenshot_path)
 
-    def is_valid(self):
+    def assert_is_valid(self):
         self.set_current_screenshot()
         if self.set_new_base:
             self.set_base_screenshot()
@@ -66,7 +72,6 @@ def is_valid(self):
             self.base_screenshot_path,
             self.current_screenshot_path
         )
-        if image_diff_result == 0:
-            return True
-        else:
-            return False
+        if image_diff_result != 0:
+            __tracebackhide__ = True
+            raise ScreenshotMismatchError("The current screenshot doesn't match the base image.")