Repository: vispy/vispy

Issue:
vispy.test() is broken for pip-installed vispy
The tarball on PyPI does not include documentation, examples, or the `make` directory. The readme does not indicate any other way of running tests, so I have no way of verifying that my installation is correct (besides running random scripts).



Discussion:
I was under the impression that PyPI packages should be minimal so they don't take up more space than they need.

@jdreaver indeed. AFAIK this is consistent with what other packages (numpy, scipy, scikit-learn, etc.) do to keep download sizes small.

You should be able to do:

``` Python
>>> import vispy
>>> vispy.test()
```

But it's currently broken. I'll change the title here as a reminder to fix it, and update the README to tell people they can run it.

For sure, leaving out documentation is understandable; I just mentioned it in case it was unintentional.

But it would definitely be good to have those tests working (even if just minimal).

I tried out `vispy.test()`, and I see this error:

```
Running tests with pyqt4 backend:
_VISPY_TESTING_APP=pyqt4 -m vispy_app_test  --cov vispy --no-cov-on-fail 
usage: -c [options] [file_or_dir] [file_or_dir] [...]
-c: error: unrecognized arguments: --cov --no-cov-on-fail
Failed: unit failure (2)
```

Now, it's obvious that I need `pytest-cov`, but I don't think it should be a requirement anywhere other than Travis (if that's possible.)

Yeah, even with `pytest-cov` installed I get an error that needs to be fixed :) But yes, I agree that shouldn't be required.

If I comment out the coverage stuff, then I get three errors in `vispy/gloo/gl/tests/test_functionality.py` (for both `pyqt4` and `pyglet` backends), other Qt backends do not run because the first one is loaded, the `nobackend` test seems to load extra tests from _other_ packages, and the remaining backends are skipped because I don't have the dependencies.


Resolving patch:
diff --git a/make/make.py b/make/make.py
--- a/make/make.py
+++ b/make/make.py
@@ -183,12 +183,15 @@ def test(self, arg):
                 * examples - run all examples
                 * examples [examples paths] - run given examples
         """
+        # Note: By default, "python make full" *will* produce coverage data,
+        # whereas vispy.test('full') will not. This is because users won't
+        # really care about coveraged, but developers will.
         if not arg:
             return self.help('test')
         from vispy import test
         try:
             args = arg.split(' ')
-            test(args[0], ' '.join(args[1:]))
+            test(args[0], ' '.join(args[1:]), coverage=True)
         except Exception as err:
             print(err)
             if not isinstance(err, RuntimeError):
diff --git a/vispy/util/logs.py b/vispy/util/logs.py
--- a/vispy/util/logs.py
+++ b/vispy/util/logs.py
@@ -4,7 +4,6 @@
 
 import base64
 import logging
-import math
 import sys
 import inspect
 import re
@@ -30,7 +29,7 @@ def _get_vispy_caller():
             line = str(record[0].f_lineno)
             func = record[3]
             cls = record[0].f_locals.get('self', None)
-            clsname = "" if cls is None else cls.__class__.__name__ + '.' 
+            clsname = "" if cls is None else cls.__class__.__name__ + '.'
             caller = "{0}:{1}{2}({3}): ".format(module, clsname, func, line)
             return caller
     return 'unknown'
@@ -314,7 +313,7 @@ def _handle_exception(ignore_callback_errors, print_callback_errors, obj,
                     ii = registry[key]
                     # Use logarithmic selection
                     # (1, 2, ..., 10, 20, ..., 100, 200, ...)
-                    if ii % (10 ** int(math.log10(ii))) == 0:
+                    if ii == (2 ** int(np.log2(ii))):
                         this_print = ii
                     else:
                         this_print = None
diff --git a/vispy/visuals/line/line.py b/vispy/visuals/line/line.py
--- a/vispy/visuals/line/line.py
+++ b/vispy/visuals/line/line.py
@@ -358,10 +358,13 @@ def draw(self, transforms):
 
         # Do we want to use OpenGL, and can we?
         GL = None
-        try:
-            import OpenGL.GL as GL
-        except ImportError:
-            pass
+        from vispy.app._default_app import default_app
+        if default_app is not None and \
+                default_app.backend_name != 'ipynb_webgl':
+            try:
+                import OpenGL.GL as GL
+            except Exception:  # can be other than ImportError sometimes
+                pass
 
         # Turn on line smooth and/or line width
         if GL:


Test patch:
diff --git a/vispy/app/tests/test_app.py b/vispy/app/tests/test_app.py
--- a/vispy/app/tests/test_app.py
+++ b/vispy/app/tests/test_app.py
@@ -58,7 +58,7 @@ def _test_callbacks(canvas):
         backend._on_mouse_scroll(_id, 1, 0)
         backend._on_mouse_motion(_id, 10, 10)
         backend._on_close(_id)
-    elif 'qt' in backend_name.lower():
+    elif any(x in backend_name.lower() for x in ('qt', 'pyside')):
         # constructing fake Qt events is too hard :(
         pass
     elif 'sdl2' in backend_name.lower():
@@ -230,13 +230,13 @@ def fake(event):
         vert = "uniform vec4 pos;\nvoid main (void) {gl_Position = pos;}"
         frag = "uniform vec4 pos;\nvoid main (void) {gl_FragColor = pos;}"
         program = Program(vert, frag)
-        #uniform = program.uniforms[0]
+        # uniform = program.uniforms[0]
         program['pos'] = [1, 2, 3, 4]
         
         vert = "attribute vec4 pos;\nvoid main (void) {gl_Position = pos;}"
         frag = "void main (void) {}"
         program = Program(vert, frag)
-        #attribute = program.attributes[0]
+        # attribute = program.attributes[0]
         program["pos"] = [1, 2, 3, 4]
         
         # use a real program
diff --git a/vispy/scene/cameras/tests/__init__.py b/vispy/scene/cameras/tests/__init__.py
new file mode 100644
diff --git a/vispy/testing/__init__.py b/vispy/testing/__init__.py
--- a/vispy/testing/__init__.py
+++ b/vispy/testing/__init__.py
@@ -46,5 +46,6 @@
                       save_testing_image, TestingCanvas, has_pyopengl,  # noqa
                       run_tests_if_main,
                       assert_is, assert_in, assert_not_in, assert_equal,
-                      assert_not_equal, assert_raises, assert_true)  # noqa
+                      assert_not_equal, assert_raises, assert_true,  # noqa
+                      raises)  # noqa
 from ._runners import test  # noqa
diff --git a/vispy/testing/_runners.py b/vispy/testing/_runners.py
--- a/vispy/testing/_runners.py
+++ b/vispy/testing/_runners.py
@@ -14,64 +14,75 @@
 
 from ..util import use_log_level, run_subprocess
 from ..util.ptime import time
-from ._testing import SkipTest, has_backend, has_application, nottest
+from ._testing import SkipTest, has_application, nottest
 
 
 _line_sep = '-' * 70
 
 
-def _get_root_dir():
-    root_dir = os.getcwd()
-    if (op.isfile(op.join(root_dir, 'setup.py')) and
-            op.isdir(op.join(root_dir, 'vispy'))):
+def _get_import_dir():
+    import_dir = op.abspath(op.join(op.dirname(__file__), '..'))
+    up_dir = op.join(import_dir, '..')
+    if (op.isfile(op.join(up_dir, 'setup.py')) and
+            op.isdir(op.join(up_dir, 'vispy')) and
+            op.isdir(op.join(up_dir, 'examples'))):
         dev = True
     else:
-        root_dir = op.abspath(op.join(op.dirname(__file__), '..', '..'))
-        dev = True if op.isfile(op.join(root_dir, 'setup.py')) else False
-    return root_dir, dev
+        dev = False
+    return import_dir, dev
 
 
 _unit_script = """
-import pytest
+try:
+    import pytest as tester
+except ImportError:
+    import nose as tester
 try:
     import faulthandler
     faulthandler.enable()
 except Exception:
     pass
 
-raise SystemExit(pytest.main(%r))
+raise SystemExit(tester.main(%r))
 """
 
 
-def _unit(mode, extra_arg_string):
+def _unit(mode, extra_arg_string, coverage=False):
     """Run unit tests using a particular mode"""
-    cwd = os.getcwd()
+    import_dir = _get_import_dir()[0]
+    cwd = op.abspath(op.join(import_dir, '..'))
+    extra_args = [''] + extra_arg_string.split(' ')
+    del extra_arg_string
+    use_pytest = False
     try:
         import pytest  # noqa, analysis:ignore
+        use_pytest = True
     except ImportError:
-        print('Skipping pytest, pytest not installed')
-        raise SkipTest()
+        try:
+            import nose  # noqa, analysis:ignore
+        except ImportError:
+            raise SkipTest('Skipping unit tests, pytest not installed')
 
     if mode == 'nobackend':
         msg = 'Running tests with no backend'
-        extra_arg_string = '-m "not vispy_app_test" ' + extra_arg_string
-        coverage = True
+        if use_pytest:
+            extra_args += ['-m', '"not vispy_app_test"']
+        else:
+            extra_args += ['-a', '"!vispy_app_test"']
     else:
-        with use_log_level('warning', print_msg=False):
-            has, why_not = has_backend(mode, out=['why_not'])
-        if not has:
-            msg = ('Skipping tests for backend %s, not found (%s)'
-                   % (mode, why_not))
-            print(_line_sep + '\n' + msg + '\n' + _line_sep + '\n')
-            raise SkipTest(msg)
         msg = 'Running tests with %s backend' % mode
-        extra_arg_string = '-m vispy_app_test ' + extra_arg_string
-        coverage = True
-    if coverage:
-        extra_arg_string += ' --cov vispy --no-cov-on-fail '
+        if use_pytest:
+            extra_args += ['-m', 'vispy_app_test']
+        else:
+            extra_args += ['-a', 'vispy_app_test']
+    if coverage and use_pytest:
+        extra_args += ['--cov', 'vispy', '--no-cov-on-fail']
     # make a call to "python" so that it inherits whatever the system
     # thinks is "python" (e.g., virtualenvs)
-    cmd = [sys.executable, '-c', _unit_script % extra_arg_string]
+    extra_arg_string = ' '.join(extra_args)
+    insert = extra_arg_string if use_pytest else extra_args
+    extra_args += [import_dir]  # positional argument
+    cmd = [sys.executable, '-c', _unit_script % insert]
     env = deepcopy(os.environ)
 
     # We want to set this for all app backends plus "nobackend" to
@@ -83,26 +94,32 @@ def _unit(mode, extra_arg_string):
                % (_line_sep, msg, env_str, extra_arg_string))
         print(msg)
     sys.stdout.flush()
-    return_code = run_subprocess(cmd, return_code=True, cwd=cwd, env=env,
-                                 stdout=None, stderr=None)[2]
+    return_code = run_subprocess(cmd, return_code=True, cwd=cwd,
+                                 env=env, stdout=None, stderr=None)[2]
     if return_code:
         raise RuntimeError('unit failure (%s)' % return_code)
-    else:
-        out_name = '.coverage.%s' % mode
+    if coverage:
+        # Running a py.test with coverage will wipe out any files that
+        # exist as .coverage or .coverage.*. It should work to pass
+        # COVERAGE_FILE env var when doing run_subprocess above, but
+        # it does not. Therefore we instead use our own naming scheme,
+        # and in Travis when we combine them, use COVERAGE_FILE with the
+        # `coverage combine` command.
+        out_name = op.join(cwd, '.vispy-coverage.%s' % mode)
         if op.isfile(out_name):
             os.remove(out_name)
-        os.rename('.coverage', out_name)
+        os.rename(op.join(cwd, '.coverage'), out_name)
 
 
 def _flake():
     """Test flake8"""
     orig_dir = os.getcwd()
-    root_dir, dev = _get_root_dir()
-    os.chdir(root_dir)
+    import_dir, dev = _get_import_dir()
+    os.chdir(op.join(import_dir, '..'))
     if dev:
         sys.argv[1:] = ['vispy', 'examples', 'make']
     else:
-        sys.argv[1:] = ['vispy']
+        sys.argv[1:] = [op.basename(import_dir)]
     sys.argv.append('--ignore=E226,E241,E265,E266,W291,W293,W503')
     sys.argv.append('--exclude=six.py,py24_ordereddict.py,glfw.py,'
                     '_proxy.py,_es2.py,_gl2.py,_pyopengl2.py,'
@@ -135,16 +152,14 @@ def _check_line_endings():
     print('Running line endings check... ')
     sys.stdout.flush()
     report = []
-    root_dir, dev = _get_root_dir()
-    if not dev:
-        root_dir = op.join(root_dir, 'vispy')
-    for dirpath, dirnames, filenames in os.walk(root_dir):
+    import_dir, dev = _get_import_dir()
+    for dirpath, dirnames, filenames in os.walk(import_dir):
         for fname in filenames:
             if op.splitext(fname)[1] in ('.pyc', '.pyo', '.so', '.dll'):
                 continue
             # Get filename
             filename = op.join(dirpath, fname)
-            relfilename = op.relpath(filename, root_dir)
+            relfilename = op.relpath(filename, import_dir)
             # Open and check
             try:
                 with open(filename, 'rb') as fid:
@@ -203,7 +218,7 @@ def _examples(fnames_str):
         Can be a space-separated list of paths to test, or an empty string to
         auto-detect and run all examples.
     """
-    root_dir, dev = _get_root_dir()
+    import_dir, dev = _get_import_dir()
     reason = None
     if not dev:
         reason = 'Cannot test examples unless in vispy git directory'
@@ -226,7 +241,7 @@ def _examples(fnames_str):
 
     else:
         fnames = [op.join(d[0], fname)
-                  for d in os.walk(op.join(root_dir, 'examples'))
+                  for d in os.walk(op.join(import_dir, '..', 'examples'))
                   for fname in d[2] if fname.endswith('.py')]
 
     fnames = sorted(fnames, key=lambda x: x.lower())
@@ -282,7 +297,7 @@ def _examples(fnames_str):
 
 
 @nottest
-def test(label='full', extra_arg_string=''):
+def test(label='full', extra_arg_string='', coverage=False):
     """Test vispy software
 
     Parameters
@@ -292,6 +307,8 @@ def test(label='full', extra_arg_string=''):
         'flake', or any backend name (e.g., 'qt').
     extra_arg_string : str
         Extra arguments to sent to ``pytest``.
+    coverage : bool
+        If True, collect coverage data.
     """
     from vispy.app.backends import BACKEND_NAMES as backend_names
     label = label.lower()
@@ -302,16 +319,18 @@ def test(label='full', extra_arg_string=''):
     if label not in known_types + backend_names:
         raise ValueError('label must be one of %s, or a backend name %s, '
                          'not \'%s\'' % (known_types, backend_names, label))
-    work_dir = _get_root_dir()[0]
-    orig_dir = os.getcwd()
     # figure out what we actually need to run
     runs = []
     if label in ('full', 'unit'):
         for backend in backend_names:
-            runs.append([partial(_unit, backend, extra_arg_string),
+            runs.append([partial(_unit, backend, extra_arg_string, coverage),
                          backend])
     elif label in backend_names:
-        runs.append([partial(_unit, label, extra_arg_string), label])
+        runs.append([partial(_unit, label, extra_arg_string, coverage), label])
+
+    if label in ('full', 'unit', 'nobackend'):
+        runs.append([partial(_unit, 'nobackend', extra_arg_string, coverage),
+                     'nobackend'])
 
     if label == "examples":
         # take the extra arguments so that specific examples can be run
@@ -321,9 +340,6 @@ def test(label='full', extra_arg_string=''):
         # run all the examples
         runs.append([partial(_examples, ""), 'examples'])
 
-    if label in ('full', 'unit', 'nobackend'):
-        runs.append([partial(_unit, 'nobackend', extra_arg_string),
-                     'nobackend'])
     if label in ('full', 'extra', 'lineendings'):
         runs.append([_check_line_endings, 'lineendings'])
     if label in ('full', 'extra', 'flake'):
@@ -333,7 +349,6 @@ def test(label='full', extra_arg_string=''):
     skip = []
     for run in runs:
         try:
-            os.chdir(work_dir)
             run[0]()
         except RuntimeError as exp:
             print('Failed: %s' % str(exp))
@@ -349,9 +364,7 @@ def test(label='full', extra_arg_string=''):
             traceback.print_exception(type_, value, tb)
         else:
             print('Passed\n')
-        finally:
-            sys.stdout.flush()
-            os.chdir(orig_dir)
+        sys.stdout.flush()
     dt = time() - t0
     stat = '%s failed, %s skipped' % (fail if fail else 0, skip if skip else 0)
     extra = 'failed' if fail else 'succeeded'
diff --git a/vispy/testing/_testing.py b/vispy/testing/_testing.py
--- a/vispy/testing/_testing.py
+++ b/vispy/testing/_testing.py
@@ -125,6 +125,25 @@ def assert_is(expr1, expr2, msg=None):
         raise AssertionError(_format_msg(msg, std_msg))
 
 
+class raises(object):
+    """Helper class to test exception raising"""
+    def __init__(self, exc):
+        self.exc = exc
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_typ, exc, tb):
+        if isinstance(exc, self.exc):
+            return True
+        elif exc is None:
+            raise AssertionError("Expected %s (no exception raised)" %
+                                 self.exc.__name__)
+        else:
+            raise AssertionError("Expected %s, got %s instead" % 
+                                 (self.exc.__name__, type(exc).__name__))
+
+
 ###############################################################################
 # GL stuff
 
diff --git a/vispy/testing/image_tester.py b/vispy/testing/image_tester.py
--- a/vispy/testing/image_tester.py
+++ b/vispy/testing/image_tester.py
@@ -57,7 +57,7 @@
 tester = None
 
 
-def get_tester():
+def _get_tester():
     global tester
     if tester is None:
         tester = ImageTester()
@@ -132,7 +132,7 @@ def assert_image_approved(image, standard_file, message=None, **kwargs):
                   "%s`\n" % (std_file, data_path, standard_file))
         if config['audit_tests']:
             sys.excepthook(*sys.exc_info())
-            get_tester().test(image, std_image, message)
+            _get_tester().test(image, std_image, message)
             std_path = os.path.dirname(std_file)
             print('Saving new standard image to "%s"' % std_file)
             if not os.path.isdir(std_path):
diff --git a/vispy/visuals/tests/__init__.py b/vispy/visuals/tests/__init__.py
new file mode 100644
diff --git a/vispy/visuals/tests/test_rectangle.py b/vispy/visuals/tests/test_rectangle.py
--- a/vispy/visuals/tests/test_rectangle.py
+++ b/vispy/visuals/tests/test_rectangle.py
@@ -7,9 +7,8 @@
 
 from vispy.scene import visuals, transforms
 from vispy.testing import (requires_application, TestingCanvas,
-                           run_tests_if_main)
+                           run_tests_if_main, raises)
 from vispy.testing.image_tester import assert_image_approved
-from pytest import raises
 
 
 @requires_application()
@@ -36,7 +35,7 @@ def test_rectangle_draw():
                                         width=80., radius=10.,
                                         border_color='white')
         c.draw_visual(rectpolygon)
-        assert_image_approved("screenshot", 'visuals/rectpolygon4.png', 
+        assert_image_approved("screenshot", 'visuals/rectpolygon4.png',
                               min_corr=0.5)
 
         rectpolygon = visuals.Rectangle(pos=(50, 50, 0), height=60.,
@@ -71,7 +70,7 @@ def test_rectpolygon_draw():
         rectpolygon.transform = transforms.STTransform(scale=(1.5, 0.5),
                                                        translate=(50, 50))
         c.draw_visual(rectpolygon)
-        assert_image_approved("screenshot", 'visuals/rectpolygon8.png', 
+        assert_image_approved("screenshot", 'visuals/rectpolygon8.png',
                               min_corr=0.5)
 
         rectpolygon = visuals.Rectangle(pos=(0., 0.), height=60.,
diff --git a/vispy/visuals/tests/test_volume.py b/vispy/visuals/tests/test_volume.py
--- a/vispy/visuals/tests/test_volume.py
+++ b/vispy/visuals/tests/test_volume.py
@@ -1,17 +1,16 @@
 # -*- coding: utf-8 -*-
 
 import numpy as np
-from pytest import raises
 from vispy import scene
 
 from vispy.testing import (TestingCanvas, requires_application,
-                           run_tests_if_main, requires_pyopengl)
+                           run_tests_if_main, requires_pyopengl,
+                           raises)
 from vispy.testing.image_tester import assert_image_approved
 
 
 @requires_pyopengl()
 def test_volume():
-    
     vol = np.zeros((20, 20, 20), 'float32')
     vol[8:16, 8:16, :] = 1.0
     
@@ -28,7 +27,7 @@ def test_volume():
     V.set_data(vol, (0.5, 0.8))
     assert V.clim == (0.5, 0.8)
     with raises(ValueError):
-        V.set_data(vol, (0.5, 0.8, 1.0))
+        V.set_data((0.5, 0.8, 1.0))
     
     # Method
     V.method = 'iso'
@@ -53,7 +52,7 @@ def test_volume_draw():
         np.random.seed(2376)
         vol = np.random.normal(size=(20, 20, 20), loc=0.5, scale=0.2)
         vol[8:16, 8:16, :] += 1.0
-        V = scene.visuals.Volume(vol, parent=v.scene)  # noqa
+        scene.visuals.Volume(vol, parent=v.scene)  # noqa
         v.camera.set_range()
         assert_image_approved(c.render(), 'visuals/volume.png')
 
diff --git a/vispy/visuals/transforms/tests/__init__.py b/vispy/visuals/transforms/tests/__init__.py
new file mode 100644