Repository: ipython/ipython

Issue:
Notebook CasperJS tests
[ not for 1.0 ] 

This PR is a starting point for testing the notebook with CasperJS.

See also:
https://github.com/ipython/ipython/wiki/IPEP-14:-Notebook-Regression-Tests



Discussion:

Resolving patch:
diff --git a/IPython/html/notebookapp.py b/IPython/html/notebookapp.py
--- a/IPython/html/notebookapp.py
+++ b/IPython/html/notebookapp.py
@@ -711,8 +711,9 @@ def cleanup_kernels(self):
 
     def notebook_info(self):
         "Return the current working directory and the server url information"
-        mgr_info = self.notebook_manager.info_string() + "\n"
-        return mgr_info +"The IPython Notebook is running at: %s" % self._url
+        info = self.notebook_manager.info_string() + "\n"
+        info += "%d active kernels \n" % len(self.kernel_manager._kernels)
+        return info + "The IPython Notebook is running at: %s" % self._url
 
     def start(self):
         """ Start the IPython Notebook server app, after initialization
diff --git a/setupbase.py b/setupbase.py
--- a/setupbase.py
+++ b/setupbase.py
@@ -137,7 +137,6 @@ def find_package_data():
     cwd = os.getcwd()
     os.chdir(os.path.join('IPython', 'html'))
     static_walk = list(os.walk('static'))
-    os.chdir(cwd)
     static_data = []
     for parent, dirs, files in static_walk:
         if parent.startswith(excludes):
@@ -145,12 +144,17 @@ def find_package_data():
         for f in files:
             static_data.append(os.path.join(parent, f))
 
+    os.chdir(os.path.join('tests',))
+    js_tests = glob('casperjs/*.*') +  glob('casperjs/*/*')
+    os.chdir(cwd)
+
     package_data = {
         'IPython.config.profile' : ['README*', '*/*.py'],
         'IPython.core.tests' : ['*.png', '*.jpg'],
         'IPython.testing' : ['*.txt'],
         'IPython.testing.plugin' : ['*.txt'],
         'IPython.html' : ['templates/*'] + static_data,
+        'IPython.html.tests' : js_tests,
         'IPython.qt.console' : ['resources/icon/*.svg'],
         'IPython.nbconvert' : ['templates/*.tpl', 'templates/latex/*.tplx',
             'templates/latex/skeleton/*.tplx', 'templates/skeleton/*', 


Test patch:
diff --git a/IPython/html/tests/casperjs/README.md b/IPython/html/tests/casperjs/README.md
new file mode 100644
--- /dev/null
+++ b/IPython/html/tests/casperjs/README.md
@@ -0,0 +1,23 @@
+# IPython Notebook Javascript Tests
+
+Regression tests for the web notebook. These tests depend on
+[CasperJS](http://casperjs.org/), which in turn requires
+a recent version of [PhantomJS](http://phantomjs.org/).
+
+Run the tests using:
+
+```
+iptest js
+```
+
+For finer granularity, or to specify more options, you can also run the
+following `casperjs` command
+
+```sh
+/path/to/bin/casperjs test --includes=util.js test_cases
+```
+
+The file `util.js` contains utility functions for tests, including a path to a
+running notebook server on localhost (http://127.0.0.1) with the port number
+specified as a command line argument to the test suite. Port 8888 is used if
+`--port=` is not specified.
diff --git a/IPython/html/tests/casperjs/test_cases/check_interrupt.js b/IPython/html/tests/casperjs/test_cases/check_interrupt.js
new file mode 100644
--- /dev/null
+++ b/IPython/html/tests/casperjs/test_cases/check_interrupt.js
@@ -0,0 +1,39 @@
+//
+// Test kernel interrupt 
+//
+casper.notebook_test(function () {
+    this.evaluate(function () {
+        var cell = IPython.notebook.get_cell(0);
+        cell.set_text('import time\nfor x in range(3):\n    time.sleep(1)');
+        cell.execute();
+    });
+
+
+    // interrupt using menu item (Kernel -> Interrupt)
+    this.thenClick('li#int_kernel');
+
+    this.wait_for_output(0);
+
+    this.then(function () {
+        var result = this.get_output_cell(0);
+        this.test.assertEquals(result.ename, 'KeyboardInterrupt', 'keyboard interrupt (mouseclick)');
+    });
+
+    // run cell 0 again, now interrupting using keyboard shortcut
+    this.thenEvaluate(function () {
+        cell.clear_output();
+        cell.execute();
+    });
+
+    // interrupt using Ctrl-M I keyboard shortcut
+    this.thenEvaluate( function() {
+        IPython.utils.press_ghetto(IPython.utils.keycodes.I)
+    });
+    
+    this.wait_for_output(0);
+    
+    this.then(function () {
+        var result = this.get_output_cell(0);
+        this.test.assertEquals(result.ename, 'KeyboardInterrupt', 'keyboard interrupt (shortcut)');
+    });
+});
diff --git a/IPython/html/tests/casperjs/test_cases/empty_nb_arrow_keys.js b/IPython/html/tests/casperjs/test_cases/empty_nb_arrow_keys.js
new file mode 100644
--- /dev/null
+++ b/IPython/html/tests/casperjs/test_cases/empty_nb_arrow_keys.js
@@ -0,0 +1,21 @@
+//
+// Check for errors with up and down arrow presses in an empty notebook.
+//
+casper.notebook_test(function () {
+    var result = this.evaluate(function() {
+        var ncells = IPython.notebook.ncells();
+        var i;
+        
+        // Delete all cells.
+        for (i = 0; i < ncells; i++) {
+            IPython.notebook.delete_cell();
+        }
+        
+        // Simulate the "up arrow" and "down arrow" keys.
+        //
+        IPython.utils.press_up();
+        IPython.utils.press_down();
+        return true;
+    });
+    this.test.assertTrue(result, 'Up/down arrow okay in empty notebook.');
+});
diff --git a/IPython/html/tests/casperjs/test_cases/execute_code_cell.js b/IPython/html/tests/casperjs/test_cases/execute_code_cell.js
new file mode 100644
--- /dev/null
+++ b/IPython/html/tests/casperjs/test_cases/execute_code_cell.js
@@ -0,0 +1,69 @@
+//
+// Test code cell execution.
+//
+casper.notebook_test(function () {
+    this.evaluate(function () {
+        var cell = IPython.notebook.get_cell(0);
+        cell.set_text('a=10; print(a)');
+        cell.execute();
+    });
+
+    this.wait_for_output(0);
+
+    // refactor this into  just a get_output(0)
+    this.then(function () {
+        var result = this.get_output_cell(0);
+        this.test.assertEquals(result.text, '10\n', 'cell execute (using js)');
+    });
+
+
+    // do it again with the keyboard shortcut
+    this.thenEvaluate(function () {
+        var cell = IPython.notebook.get_cell(0);
+        cell.set_text('a=11; print(a)');
+        cell.clear_output();
+        IPython.utils.press_ctrl_enter();
+    });
+
+    this.wait_for_output(0);
+
+    this.then(function () {
+        var result = this.get_output_cell(0);
+        var num_cells = this.get_cells_length();
+        this.test.assertEquals(result.text, '11\n', 'cell execute (using ctrl-enter)');
+        this.test.assertEquals(num_cells, 1, '              ^--- does not add a new cell')
+    });
+    
+    // do it again with the keyboard shortcut
+    this.thenEvaluate(function () {
+        var cell = IPython.notebook.get_cell(0);
+        cell.set_text('a=12; print(a)');
+        cell.clear_output();
+        IPython.utils.press_shift_enter();
+    });
+
+    this.wait_for_output(0);
+
+    this.then(function () {
+        var result = this.get_output_cell(0);
+        var num_cells = this.get_cells_length();
+        this.test.assertEquals(result.text, '12\n', 'cell execute (using shift-enter)');
+        this.test.assertEquals(num_cells, 2, '              ^--- adds a new cell')
+    });
+
+    // press the "play" triangle button in the toolbar
+    this.thenEvaluate(function () {
+        var cell = IPython.notebook.get_cell(0);
+        IPython.notebook.select(0);
+        cell.clear_output();
+        cell.set_text('a=13; print(a)');
+        $('#run_b').click();
+    });
+    
+    this.wait_for_output(0);
+
+    this.then(function () {
+        var result = this.get_output_cell(0);
+        this.test.assertEquals(result.text, '13\n', 'cell execute (using "play" toolbar button)')
+    });
+});
diff --git a/IPython/html/tests/casperjs/test_cases/merge_cells.js b/IPython/html/tests/casperjs/test_cases/merge_cells.js
new file mode 100644
--- /dev/null
+++ b/IPython/html/tests/casperjs/test_cases/merge_cells.js
@@ -0,0 +1,37 @@
+//
+// Test merging two notebook cells.
+//
+casper.notebook_test(function() {
+    var output = this.evaluate(function () {
+        // Fill in test data.
+        var set_cell_text = function () {
+            var cell_one = IPython.notebook.get_selected_cell();
+            cell_one.set_text('a = 5');
+            
+            IPython.notebook.insert_cell_below('code');
+            var cell_two = IPython.notebook.get_selected_cell();
+            cell_two.set_text('print(a)');
+        };
+        
+        // merge_cell_above()
+        set_cell_text();
+        IPython.notebook.merge_cell_above();
+        var merged_above = IPython.notebook.get_selected_cell();
+        
+        // merge_cell_below()
+        set_cell_text();
+        IPython.notebook.select(0);
+        IPython.notebook.merge_cell_below();
+        var merged_below = IPython.notebook.get_selected_cell();
+        
+        return {
+            above: merged_above.get_text(),
+            below: merged_below.get_text()
+        };
+    });
+    
+    this.test.assertEquals(output.above, 'a = 5\nprint(a)',
+                           'Successful insert_cell_above().');
+    this.test.assertEquals(output.below, 'a = 5\nprint(a)',
+                           'Successful insert_cell_below().');
+});
diff --git a/IPython/html/tests/casperjs/test_cases/nb_arrow_keys.js b/IPython/html/tests/casperjs/test_cases/nb_arrow_keys.js
new file mode 100644
--- /dev/null
+++ b/IPython/html/tests/casperjs/test_cases/nb_arrow_keys.js
@@ -0,0 +1,23 @@
+//
+// Check for errors with up and down arrow presses in a non-empty notebook.
+//
+casper.notebook_test(function () {
+    var result = this.evaluate(function() {
+        pos0 = IPython.notebook.get_selected_index();
+        IPython.notebook.insert_cell_below('code'); 
+        pos1 = IPython.notebook.get_selected_index();
+        IPython.notebook.insert_cell_below('code'); 
+        pos2 = IPython.notebook.get_selected_index();
+        // Simulate the "up arrow" and "down arrow" keys.
+        IPython.utils.press_up();
+        pos3 = IPython.notebook.get_selected_index();
+        IPython.utils.press_down();
+        pos4 = IPython.notebook.get_selected_index();
+        return  pos0 == 0 &&
+                pos1 == 1 &&
+                pos2 == 2 &&
+                pos3 == 1 &&
+                pos4 == 2;
+    });
+    this.test.assertTrue(result, 'Up/down arrow okay in non-empty notebook.');
+});
diff --git a/IPython/html/tests/casperjs/test_cases/render_markdown.js b/IPython/html/tests/casperjs/test_cases/render_markdown.js
new file mode 100644
--- /dev/null
+++ b/IPython/html/tests/casperjs/test_cases/render_markdown.js
@@ -0,0 +1,36 @@
+//
+// Test that a Markdown cell is rendered to HTML.
+//
+casper.notebook_test(function () {
+    // Test JavaScript models.
+    var output = this.evaluate(function () {
+        IPython.notebook.to_markdown();
+        var cell = IPython.notebook.get_selected_cell();
+        cell.set_text('# Foo');
+        cell.render();
+        return cell.get_rendered();
+    });
+    this.test.assertEquals(output, '<h1>Foo</h1>', 'Markdown JS API works.');
+    
+    // Test menubar entries.
+    output = this.evaluate(function () {
+        $('#to_code').mouseenter().click();
+        $('#to_markdown').mouseenter().click();
+        var cell = IPython.notebook.get_selected_cell();
+        cell.set_text('# Foo');
+        $('#run_cell').mouseenter().click();
+        return cell.get_rendered();
+    });
+    this.test.assertEquals(output, '<h1>Foo</h1>', 'Markdown menubar items work.');
+    
+    // Test toolbar buttons.
+    output = this.evaluate(function () {
+        $('#cell_type').val('code').change();
+        $('#cell_type').val('markdown').change();
+        var cell = IPython.notebook.get_selected_cell();
+        cell.set_text('# Foo');
+        $('#run_b').click();
+        return cell.get_rendered();
+    });
+    this.test.assertEquals(output, '<h1>Foo</h1>', 'Markdown toolbar items work.');
+});
diff --git a/IPython/html/tests/casperjs/test_cases/shutdown_notebook.js b/IPython/html/tests/casperjs/test_cases/shutdown_notebook.js
new file mode 100644
--- /dev/null
+++ b/IPython/html/tests/casperjs/test_cases/shutdown_notebook.js
@@ -0,0 +1,31 @@
+//
+// Test shutdown of a kernel.
+//
+casper.notebook_test(function () {
+    // XXX: test.begin allows named sections but requires casperjs 1.1.0-DEV.
+    //      We will put it back into place when the next version of casper is
+    //      released. Following that, all instances of this.test can be changed
+    //      to just test.
+    //this.test.begin("shutdown tests (notebook)", 2, function(test) {
+
+    this.thenEvaluate(function () {
+        $('#kill_and_exit').click();
+    });
+    
+    this.thenEvaluate(function () {
+        var cell = IPython.notebook.get_cell(0);
+        cell.set_text('a=10; print(a)');
+        cell.execute();
+    });
+
+    // refactor this into  just a get_output(0)
+    this.then(function () {
+        var result = this.get_output_cell(0);
+        this.test.assertFalsy(result, "after shutdown: no execution results");
+        this.test.assertNot(this.kernel_running(), 
+            'after shutdown: IPython.notebook.kernel.running is false ');
+    });
+
+//}); // end of test.begin
+});
+
diff --git a/IPython/html/tests/casperjs/util.js b/IPython/html/tests/casperjs/util.js
new file mode 100644
--- /dev/null
+++ b/IPython/html/tests/casperjs/util.js
@@ -0,0 +1,123 @@
+//
+// Utility functions for the HTML notebook's CasperJS tests.
+//
+
+// Get the URL of a notebook server on which to run tests.
+casper.get_notebook_server = function () {
+    port = casper.cli.get("port")
+    port = (typeof port === 'undefined') ? '8888' : port;
+    return 'http://127.0.0.1:' + port
+};
+
+// Create and open a new notebook.
+casper.open_new_notebook = function () {
+    var baseUrl = this.get_notebook_server();
+    this.start(baseUrl);
+    this.thenClick('button#new_notebook');
+    this.waitForPopup('');
+
+    this.withPopup('', function () {this.waitForSelector('.CodeMirror-code');});
+    this.then(function () {
+        // XXX: Kind of odd, the next line works for one test, but not when
+        // running multiple tests back-to-back, so we will just point the main
+        // casper browser to the same URL as the popup we just grabbed.
+
+        //this.page = this.popups[0];
+        this.open(this.popups[0].url);
+    });
+
+    // initially, the cells aren't created, so wait for them to appear
+    this.waitForSelector('.CodeMirror-code');
+    // and make sure the kernel has started
+    this.waitFor( this.kernel_running  );
+};
+
+// return whether or not the kernel is running
+casper.kernel_running = function kernel_running() {
+    return this.evaluate(function kernel_running() {
+        return IPython.notebook.kernel.running;
+    });
+};
+
+// Shut down the current notebook's kernel.
+casper.shutdown_current_kernel = function () {
+    this.thenEvaluate(function() {
+        IPython.notebook.kernel.kill();
+    });
+};
+
+// Delete created notebook.
+casper.delete_current_notebook = function () {
+    this.thenEvaluate(function() {
+        var nbData = $('body').data();
+        var url = nbData.baseProjectUrl + 'notebooks/' + nbData.notebookId;
+        $.ajax(url, {
+            type: 'DELETE',
+        });
+    });
+};
+
+// wait for output in a given cell
+casper.wait_for_output = function (cell_num) {
+    this.then(function() {
+        this.waitFor(function (c) {
+            return this.evaluate(function get_output(c) {
+                var cell = IPython.notebook.get_cell(c);
+                return cell.output_area.outputs.length != 0;
+            },
+            // pass parameter from the test suite js to the browser code js
+            {c : cell_num});
+        });
+    });
+};
+
+// return the output of a given cell
+casper.get_output_cell = function (cell_num) {
+    var result = casper.evaluate(function (c) {
+        var cell = IPython.notebook.get_cell(c);
+        return cell.output_area.outputs[0];
+    },
+    {c : cell_num});
+    return result;
+};
+
+// return the number of cells in the notebook
+casper.get_cells_length = function () {
+    var result = casper.evaluate(function () {
+        return IPython.notebook.get_cells().length;
+    })
+    return result;
+};
+
+// Wrap a notebook test to reduce boilerplate.
+casper.notebook_test = function(test) {
+    this.open_new_notebook();
+    this.then(test);
+    //XXX: we get sporadic error messages when shutting down some of the tests.
+    //     Since the entire server will go down at the end of running the test
+    //     suite, it's ok for now to not try to shut anything down.
+    this.shutdown_current_kernel();
+    
+    //XXX: the implementation of delete_current_notebook is currently broken
+    //     it's not a big deal, since the notebook directory will be deleted on
+    //     cleanup, but we should add tests for deleting the notebook separately
+    //this.delete_current_notebook();
+    
+    // Run the browser automation.
+    this.run(function() {
+        this.test.done();
+    });
+};
+
+casper.options.waitTimeout=10000
+casper.on('waitFor.timeout', function onWaitForTimeout(timeout) {
+    this.echo("Timeout for " + casper.get_notebook_server());
+    this.echo("Is the notebook server running?");
+});
+
+// Pass `console.log` calls from page JS to casper.
+casper.printLog = function () {
+    this.on('remote.message', function(msg) {
+        this.echo('Remote message caught: ' + msg);
+    });
+};
diff --git a/IPython/testing/iptest.py b/IPython/testing/iptest.py
--- a/IPython/testing/iptest.py
+++ b/IPython/testing/iptest.py
@@ -45,6 +45,7 @@
 from nose.util import safe_str
 
 # Our own imports
+from IPython.utils.process import is_cmd_found
 from IPython.utils.importstring import import_item
 from IPython.testing.plugin.ipdoctest import IPythonDoctest
 from IPython.external.decorators import KnownFailure, knownfailureif
@@ -151,6 +152,7 @@ def test_for(item, min_version=None, callback=extract_version):
 have['azure'] = test_for('azure')
 have['requests'] = test_for('requests')
 have['sphinx'] = test_for('sphinx')
+have['casperjs'] = is_cmd_found('casperjs')
 
 min_zmq = (2,1,11)
 
diff --git a/IPython/testing/iptestcontroller.py b/IPython/testing/iptestcontroller.py
--- a/IPython/testing/iptestcontroller.py
+++ b/IPython/testing/iptestcontroller.py
@@ -20,6 +20,7 @@
 
 import argparse
 import multiprocessing.pool
+from multiprocessing import Process, Queue
 import os
 import shutil
 import signal
@@ -27,7 +28,7 @@
 import subprocess
 import time
 
-from .iptest import have, test_group_names, test_sections
+from .iptest import have, test_group_names as py_test_group_names, test_sections
 from IPython.utils.py3compat import bytes_to_str
 from IPython.utils.sysinfo import sys_info
 from IPython.utils.tempdir import TemporaryDirectory
@@ -55,11 +56,6 @@ def __init__(self):
         self.cmd = []
         self.env = {}
         self.dirs = []
-    
-    @property
-    def will_run(self):
-        """Override in subclasses to check for dependencies."""
-        return False
 
     def launch(self):
         # print('*** ENV:', self.env)  # dbg
@@ -98,13 +94,13 @@ def cleanup_process(self):
         if subp.poll() is None:
             # The process did not die...
             print('... failed. Manual cleanup may be required.')
-    
+
     def cleanup(self):
         "Kill process if it's still alive, and clean up temporary directories"
         self.cleanup_process()
         for td in self.dirs:
             td.cleanup()
-    
+
     __del__ = cleanup
 
 class PyTestController(TestController):
@@ -162,22 +158,91 @@ def launch(self):
         self.cmd[2] = self.pycmd
         super(PyTestController, self).launch()
 
+class JSController(TestController):
+    """Run CasperJS tests """
+    def __init__(self, section):
+        """Create new test runner."""
+        TestController.__init__(self)
+        self.section = section
+
+        self.ipydir = TemporaryDirectory()
+        self.dirs.append(self.ipydir)
+        self.env['IPYTHONDIR'] = self.ipydir.name
 
-def prepare_py_test_controllers(inc_slow=False):
-    """Returns an ordered list of PyTestController instances to be run."""
-    to_run, not_run = [], []
-    if not inc_slow:
-        test_sections['parallel'].enabled = False
+    def launch(self):
+        # start the ipython notebook, so we get the port number
+        self._init_server()
 
-    for name in test_group_names:
-        controller = PyTestController(name)
-        if controller.will_run:
-            to_run.append(controller)
-        else:
-            not_run.append(controller)
+        import IPython.html.tests as t
+        test_dir = os.path.join(os.path.dirname(t.__file__), 'casperjs')
+        includes = '--includes=' + os.path.join(test_dir,'util.js')
+        test_cases = os.path.join(test_dir, 'test_cases')
+        port = '--port=' + str(self.server_port)
+        self.cmd = ['casperjs', 'test', port, includes, test_cases]
+
+        super(JSController, self).launch()
+
+    @property
+    def will_run(self):
+        return all(have[a] for a in ['zmq', 'tornado', 'jinja2', 'casperjs'])
+
+    def _init_server(self):
+        "Start the notebook server in a separate process"
+        self.queue = q = Queue()
+        self.server = Process(target=run_webapp, args=(q, self.ipydir.name))
+        self.server.start()
+        self.server_port = q.get()
+
+    def cleanup(self):
+        self.server.terminate()
+        self.server.join()
+        TestController.cleanup(self)
+
+js_test_group_names = {'js'}
+
+def run_webapp(q, nbdir, loglevel=0):
+    """start the IPython Notebook, and pass port back to the queue"""
+    import os
+    import IPython.html.notebookapp as nbapp
+    import sys
+    sys.stderr = open(os.devnull, 'w')
+    os.environ["IPYTHONDIR"] = nbdir
+    server = nbapp.NotebookApp()
+    args = ['--no-browser']
+    args.append('--notebook-dir='+nbdir)
+    args.append('--profile-dir='+nbdir)
+    args.append('--log-level='+str(loglevel))
+    server.initialize(args)
+    # communicate the port number to the parent process
+    q.put(server.port)
+    server.start()
+
+def prepare_controllers(options):
+    """Returns two lists of TestController instances, those to run, and those
+    not to run."""
+    testgroups = options.testgroups
+
+    if testgroups:
+        py_testgroups = [g for g in testgroups if g in py_test_group_names]
+        js_testgroups = [g for g in testgroups if g in js_test_group_names]
+    else:
+        py_testgroups = py_test_group_names
+        js_testgroups = js_test_group_names
+        if not options.all:
+            test_sections['parallel'].enabled = False
+
+    c_js = [JSController(name) for name in js_testgroups]
+    c_py = [PyTestController(name) for name in py_testgroups]
+
+    configure_py_controllers(c_py, xunit=options.xunit,
+            coverage=options.coverage)
+
+    controllers = c_py + c_js
+    to_run = [c for c in controllers if c.will_run]
+    not_run = [c for c in controllers if not c.will_run]
     return to_run, not_run
 
-def configure_controllers(controllers, xunit=False, coverage=False, extra_args=()):
+def configure_py_controllers(controllers, xunit=False, coverage=False, extra_args=()):
     """Apply options for a collection of TestController objects."""
     for controller in controllers:
         if xunit:
@@ -194,10 +259,10 @@ def do_run(controller):
             import traceback
             traceback.print_exc()
             return controller, 1  # signal failure
-    
+
         exitcode = controller.wait()
         return controller, exitcode
-    
+
     except KeyboardInterrupt:
         return controller, -signal.SIGINT
     finally:
@@ -236,7 +301,7 @@ def run_iptestall(options):
     modules and package and then runs each of them.  This causes the modules
     and packages of IPython to be tested each in their own subprocess using
     nose.
-    
+
     Parameters
     ----------
 
@@ -268,14 +333,7 @@ def run_iptestall(options):
         # If running in parallel, capture output so it doesn't get interleaved
         TestController.buffer_output = True
 
-    if options.testgroups:
-        to_run = [PyTestController(name) for name in options.testgroups]
-        not_run = []
-    else:
-        to_run, not_run = prepare_py_test_controllers(inc_slow=options.all)
-
-    configure_controllers(to_run, xunit=options.xunit, coverage=options.coverage,
-                          extra_args=options.extra_args)
+    to_run, not_run = prepare_controllers(options)
 
     def justify(ltext, rtext, width=70, fill='-'):
         ltext += ' '
@@ -315,7 +373,7 @@ def justify(ltext, rtext, width=70, fill='-'):
                         break
         except KeyboardInterrupt:
             return
-    
+
     for controller in not_run:
         print(justify('IPython test group: ' + controller.section, 'NOT RUN'))