Repository: bokeh/bokeh

Issue:
Enable exclusion of selenium tests for static image export
Running Bokeh unit tests on 0.12.6.dev7...

py.test -m "not (examples or integration or js or quality)" bokeh

reports 3 failed tests. In discussion with @bryevdv looks like they are selenium tests for the new static image export feature. Would be useful to add a new mark for them so that they can be excluded easily.


Discussion:
@canavandl should we make a new tag for this, or add it to `integration` ?
Some projects, e.g. have a `slow-tests` tag for a subset of unit tests. I don't mind adding it to `integration` though, either. 

It would be nice to also see if that plugin to make a "negative" or "unlabeled" category works or not. I'd like to be able to do `py.test -m unit` and get everything that is not otherwise tagged. 
I think I will try to add support for `pytest-unmarked` as well. That was previously closed as `wontfix` in https://github.com/bokeh/bokeh/issues/3906 because the plugin used a private `pytest `API but the list of things to exclude to get the plain unit tests is getting too long. If it stops working at some point there's actually minimal effect except to local usage. 

Resolving patch:
diff --git a/conftest.py b/conftest.py
new file mode 100644
--- /dev/null
+++ b/conftest.py
@@ -0,0 +1 @@
+pytest_plugins = "tests.plugins.implicit_mark"


Test patch:
diff --git a/bokeh/command/subcommands/tests/test_png.py b/bokeh/command/subcommands/tests/test_png.py
--- a/bokeh/command/subcommands/tests/test_png.py
+++ b/bokeh/command/subcommands/tests/test_png.py
@@ -67,6 +67,8 @@ def test_no_script(capsys):
 """ % (too_few)
         assert out == ""
 
+@pytest.mark.unit
+@pytest.mark.selenium
 def test_basic_script(capsys):
     def run(dirname):
         with WorkingDir(dirname):
@@ -80,6 +82,8 @@ def run(dirname):
     with_directory_contents({ 'scatter.py' : basic_scatter_script },
                             run)
 
+@pytest.mark.unit
+@pytest.mark.selenium
 def test_basic_script_with_output_after(capsys):
     def run(dirname):
         with WorkingDir(dirname):
@@ -93,6 +97,8 @@ def run(dirname):
     with_directory_contents({ 'scatter.py' : basic_scatter_script },
                             run)
 
+@pytest.mark.unit
+@pytest.mark.selenium
 def test_basic_script_with_output_before(capsys):
     def run(dirname):
         with WorkingDir(dirname):
diff --git a/bokeh/command/subcommands/tests/test_svg.py b/bokeh/command/subcommands/tests/test_svg.py
--- a/bokeh/command/subcommands/tests/test_svg.py
+++ b/bokeh/command/subcommands/tests/test_svg.py
@@ -67,6 +67,8 @@ def test_no_script(capsys):
 """ % (too_few)
         assert out == ""
 
+@pytest.mark.unit
+@pytest.mark.selenium
 def test_basic_script(capsys):
     def run(dirname):
         with WorkingDir(dirname):
@@ -80,6 +82,8 @@ def run(dirname):
     with_directory_contents({ 'scatter.py' : basic_svg_scatter_script },
                             run)
 
+@pytest.mark.unit
+@pytest.mark.selenium
 def test_basic_script_with_output_after(capsys):
     def run(dirname):
         with WorkingDir(dirname):
@@ -93,6 +97,8 @@ def run(dirname):
     with_directory_contents({ 'scatter.py' : basic_svg_scatter_script },
                             run)
 
+@pytest.mark.unit
+@pytest.mark.selenium
 def test_basic_script_with_output_before(capsys):
     def run(dirname):
         with WorkingDir(dirname):
diff --git a/bokeh/tests/test_io.py b/bokeh/tests/test_io.py
--- a/bokeh/tests/test_io.py
+++ b/bokeh/tests/test_io.py
@@ -305,6 +305,8 @@ def test__crop_image():
     cropped = io._crop_image(image, **rect)
     assert cropped.size == (6,4)
 
+@pytest.mark.unit
+@pytest.mark.selenium
 def test__get_screenshot_as_png():
     layout = Plot(x_range=Range1d(), y_range=Range1d(),
                   plot_height=20, plot_width=20, toolbar_location=None,
@@ -316,6 +318,8 @@ def test__get_screenshot_as_png():
     # a 20x20px image of transparent pixels
     assert png.tobytes() == ("\x00"*1600).encode()
 
+@pytest.mark.unit
+@pytest.mark.selenium
 def test__get_screenshot_as_png_with_driver():
     layout = Plot(x_range=Range1d(), y_range=Range1d(),
                   plot_height=20, plot_width=20, toolbar_location=None,
@@ -333,6 +337,8 @@ def test__get_screenshot_as_png_with_driver():
     # a 20x20px image of transparent pixels
     assert png.tobytes() == ("\x00"*1600).encode()
 
+@pytest.mark.unit
+@pytest.mark.selenium
 def test__get_svgs_no_svg_present():
     layout = Plot(x_range=Range1d(), y_range=Range1d(),
               plot_height=20, plot_width=20, toolbar_location=None)
@@ -340,6 +346,8 @@ def test__get_svgs_no_svg_present():
     svgs = io._get_svgs(layout, None)
     assert svgs == []
 
+@pytest.mark.unit
+@pytest.mark.selenium
 def test__get_svgs_with_svg_present():
     layout = Plot(x_range=Range1d(), y_range=Range1d(),
                   plot_height=20, plot_width=20, toolbar_location=None,
@@ -351,6 +359,8 @@ def test__get_svgs_with_svg_present():
                        'width="20" height="20" style="width: 20px; height: 20px;"><defs/><g><g/><g transform="scale(1,1) '
                        'translate(0.5,0.5)"><rect fill="#FFFFFF" stroke="none" x="0" y="0" width="20" height="20"/><g/><g/><g/><g/></g></g></svg>')
 
+@pytest.mark.unit
+@pytest.mark.selenium
 def test__get_svgs_with_svg_present_with_driver():
     layout = Plot(x_range=Range1d(), y_range=Range1d(),
                   plot_height=20, plot_width=20, toolbar_location=None,
diff --git a/sphinx/source/docs/dev_guide/testing.rst b/sphinx/source/docs/dev_guide/testing.rst
--- a/sphinx/source/docs/dev_guide/testing.rst
+++ b/sphinx/source/docs/dev_guide/testing.rst
@@ -10,14 +10,19 @@ file descriptors as some of our tests open many files to test the server.
 
     ulimit -n 1024
 
-To run just the python unit tests, run either command:
+To run all the basic python unit tests, run the following command at the top
+level of the repository:
 
 .. code-block:: sh
 
-    py.test -m 'not (js or examples or integration or quality)'
+    py.test -m unit
 
-    python -c 'import bokeh; bokeh.test()'
+Note that this includes unit tests that require Selenium to be installed. To
+exclude those unit tests, you can run the command:
 
+.. code-block:: sh
+
+    py.test -m "unit and not selenium"
 
 To run just the BokehJS unit tests, execute:
 
@@ -25,7 +30,8 @@ To run just the BokehJS unit tests, execute:
 
     py.test -m js
 
-Or, in the `bokehjs` subdirectory of the source checkout.
+Alternatively, you can also navigate to the `bokehjs` subdirectory of the
+source checkout and execute:
 
 .. code-block:: sh
 
diff --git a/tests/conftest.py b/tests/conftest.py
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -6,10 +6,10 @@
     "tests.examples.examples_report_plugin",
     "tests.integration.integration_tests_plugin",
     "tests.plugins.bokeh_server",
-    "tests.plugins.jupyter_notebook",
-    "tests.plugins.phantomjs_screenshot",
     "tests.plugins.image_diff",
+    "tests.plugins.jupyter_notebook",
     "tests.plugins.file_server",
+    "tests.plugins.phantomjs_screenshot",
 )
 
 
diff --git a/tests/plugins/implicit_mark.py b/tests/plugins/implicit_mark.py
new file mode 100644
--- /dev/null
+++ b/tests/plugins/implicit_mark.py
@@ -0,0 +1,25 @@
+from _pytest.mark import matchmark
+
+def pytest_addoption(parser):
+    parser.addini("implicit_marker",
+                  "An implicit marker to assign to any test otherwise unmarked")
+
+def pytest_collection_modifyitems(items, config):
+    implicit_marker = config.getini("implicit_marker")
+    if not implicit_marker:
+        return
+
+    markers = []
+    for line in config.getini("markers"):
+        mark, rest = line.split(":", 1)
+        if '(' in mark:
+            mark, rest = mark.split("(", 1)
+        markers.append(mark)
+
+    all_markers = ' or '.join(markers)
+    if not all_markers:
+        return
+
+    for item in items:
+        if not matchmark(item, all_markers):
+            item.add_marker(implicit_marker)