Repository: nose-devs/nose2

Issue:
MP plugin does't call startSubprocess in subprocess
My very quick diagnosis is:

In [mp.py](https://github.com/nose-devs/nose2/blob/master/nose2/plugins/mp.py):
- The new hook methods are registered on [line 27](https://github.com/nose-devs/nose2/blob/master/nose2/plugins/mp.py#L27)
- However that is only called in the master process
- Hence, when in the subprocess on [line 197](https://github.com/nose-devs/nose2/blob/master/nose2/plugins/mp.py#L197) we register the plugins, the session doesn't know to look for `startSubprocess` (or `stopSubprocess` for that matter)
- Hence when it tries to dispatch it on [line 202](https://github.com/nose-devs/nose2/blob/master/nose2/plugins/mp.py#L202) it won't call the plugins


Discussion:
coveralls: 
[![Coverage Status](https://coveralls.io/builds/14156452/badge)](https://coveralls.io/builds/14156452)

Coverage increased (+0.2%) to 87.69% when pulling **d436976ece996497dac644fc59a40b2ead323ae7 on ptthiem:issue/167** into **862b130652b9118eb8d5681923c04edb000245d3 on nose-devs:master**.

ptthiem: There is a small warning being thrown by the test, but it seems to be working.  I'll check it tomorrow afternoon
coveralls: 
[![Coverage Status](https://coveralls.io/builds/14157959/badge)](https://coveralls.io/builds/14157959)

Coverage increased (+0.07%) to 87.527% when pulling **d436976ece996497dac644fc59a40b2ead323ae7 on ptthiem:issue/167** into **862b130652b9118eb8d5681923c04edb000245d3 on nose-devs:master**.

coveralls: 
[![Coverage Status](https://coveralls.io/builds/14158069/badge)](https://coveralls.io/builds/14158069)

Coverage increased (+0.2%) to 87.636% when pulling **6a73c8057173469da7eae5a0d4c9b0a07e9867bf on ptthiem:issue/167** into **862b130652b9118eb8d5681923c04edb000245d3 on nose-devs:master**.


Patch:
diff --git a/nose2/plugins/layers.py b/nose2/plugins/layers.py
index 43025ca..eda14dc 100644
--- a/nose2/plugins/layers.py
+++ b/nose2/plugins/layers.py
@@ -235,6 +235,9 @@ class LayerReporter(events.Plugin):
         if event.errorList and hasattr(event.test, 'layer'):
             # walk back layers to build full description
             self.describeLayers(event)
+        # we need to remove "\n" from description to keep a well indented report when tests have docstrings
+        # see https://github.com/nose-devs/nose2/issues/327 for more information
+        event.description = event.description.replace('\n', ' ')
 
     def describeLayers(self, event):
         desc = [event.description]
diff --git a/nose2/plugins/mp.py b/nose2/plugins/mp.py
index a8346d0..b712bb3 100644
--- a/nose2/plugins/mp.py
+++ b/nose2/plugins/mp.py
@@ -80,8 +80,8 @@ class MultiProcess(events.Plugin):
             if testid.startswith(failed_import_id):
                 self.cases[testid].run(result_)
 
-        # XXX The length of the filtered list needs to be known
-        # for _startProcs, until this can be cleaned up.  This
+        # XXX Process-Handling: The length of the filtered list needs to be
+        # known for _startProcs, until this can be cleaned up.  This
         # wasn't the best way to deal with too few tests
         flat = [x for x in flat if not x.startswith(failed_import_id)]
         procs = self._startProcs(len(flat))
@@ -91,22 +91,26 @@ class MultiProcess(events.Plugin):
             if not flat:
                 break
             caseid = flat.pop(0)
+            # NOTE: it throws errors on broken pipes and bad serialization
             conn.send(caseid)
 
         rdrs = [conn for proc, conn in procs if proc.is_alive()]
         while flat or rdrs:
             ready, _, _ = select.select(rdrs, [], [], self.testRunTimeout)
             for conn in ready:
-                # XXX proc could be dead
+                # XXX Process-Handling: If we get an EOFError on receive the
+                # process finished= or we lost the process and the test it was
+                # working on.  Also do we rebuild the process?
                 try:
                     remote_events = conn.recv()
                 except EOFError:
                     # probably dead/12
                     log.warning("Subprocess connection closed unexpectedly")
-                    continue  # XXX or die?
+                    continue
 
+                # If remote_events is None, the process exited normally,
+                # which should mean that we didn't any more tests for it.
                 if remote_events is None:
-                    # XXX proc is done, how to mark it dead?
                     log.debug("Conn closed %s", conn)
                     rdrs.remove(conn)
                     continue
@@ -119,9 +123,10 @@ class MultiProcess(events.Plugin):
                     self._localize(event)
                     getattr(self.session.hooks, hook)(event)
 
-                # send a new test to the worker if there is one left
+                # Send the next test_id
+                # NOTE: send throws errors on broken pipes and bad serialization
                 if not flat:
-                    # if there isn't send None - it's the 'done' flag
+                    # If there are no more, send None - it's the 'done' flag
                     conn.send(None)
                     continue
                 caseid = flat.pop(0)
@@ -129,6 +134,7 @@ class MultiProcess(events.Plugin):
 
         for _, conn in procs:
             conn.close()
+
         # ensure we wait until all processes are done before
         # exiting, to allow plugins running there to finalize
         for proc, _ in procs:
@@ -174,7 +180,7 @@ class MultiProcess(events.Plugin):
             return parent_conn
 
     def _startProcs(self, test_count):
-        # XXX create session export
+        # Create session export
         session_export = self._exportSession()
         procs = []
         count = min(test_count, self.procs)
@@ -190,11 +196,15 @@ class MultiProcess(events.Plugin):
         return procs
 
     def _flatten(self, suite):
-        # XXX
-        # examine suite tests to find out if they have class
-        # or module fixtures and group them that way into names
-        # of test classes or modules
-        # ALSO record all test cases in self.cases
+        """
+        Flatten test-suite into list of IDs, AND record all test case
+        into self.cases
+
+        CAVEAT:  Due to current limitation of the MP plugin, examine the suite
+                 tests to find out if they have class or module fixtures and
+                 group them that way into name of test classes or module.
+                 This is aid in their dispatch.
+        """
         log.debug("Flattening test into list of IDs")
         mods = {}
         classes = {}
@@ -244,19 +254,27 @@ class MultiProcess(events.Plugin):
                                  "main process" % event.test))._tests[0]
 
     def _exportSession(self):
-        # argparse isn't pickleable
-        # no plugin instances
-        # no hooks
+        """
+        Generate the session information passed to work process.
+
+        CAVEAT: The entire contents of which *MUST* be pickeable
+        and safe to use in the subprocess.
+
+        This probably includes:
+        * No argparse namespaces/named-tuples
+        * No plugin instances
+        * No hokes
+        :return:
+        """
         export = {'config': self.session.config,
                   'verbosity': self.session.verbosity,
                   'startDir': self.session.startDir,
                   'topLevelDir': self.session.topLevelDir,
                   'logLevel': self.session.logLevel,
-                  # XXX classes or modules?
                   'pluginClasses': []}
-        # XXX fire registerInSubprocess -- add those plugin classes
-        # (classes must be pickleable!)
-        event = RegisterInSubprocessEvent()  # FIXME should be own event type
+        event = RegisterInSubprocessEvent()
+        # fire registerInSubprocess on plugins -- add those plugin classes
+        # CAVEAT: classes must be pickleable!
         self.session.hooks.registerInSubprocess(event)
         export['pluginClasses'].extend(event.pluginClasses)
         return export
@@ -268,31 +286,16 @@ def procserver(session_export, conn):
     rlog.setLevel(session_export['logLevel'])
 
     # make a real session from the "session" we got
-    ssn = session.Session()
-    ssn.config = session_export['config']
-    ssn.hooks = RecordingPluginInterface()
-    ssn.verbosity = session_export['verbosity']
-    ssn.startDir = session_export['startDir']
-    ssn.topLevelDir = session_export['topLevelDir']
-    ssn.prepareSysPath()
-    loader_ = loader.PluggableTestLoader(ssn)
-    ssn.testLoader = loader_
-    result_ = result.PluggableTestResult(ssn)
-    ssn.testResult = result_
-    runner_ = runner.PluggableTestRunner(ssn)  # needed??
-    ssn.testRunner = runner_
-    # load and register plugins
-    ssn.plugins = [
-        plugin(session=ssn) for plugin in session_export['pluginClasses']]
-    rlog.debug("Plugins loaded: %s", ssn.plugins)
-    for plugin in ssn.plugins:
-        plugin.register()
-        rlog.debug("Registered %s in subprocess", plugin)
+    ssn = import_session(rlog, session_export)
 
     if isinstance(conn, collections.Sequence):
         conn = connection.Client(conn[:2], authkey=conn[2])
 
-    event = SubprocessEvent(loader_, result_, runner_, ssn.plugins, conn)
+    event = SubprocessEvent(ssn.testLoader,
+                            ssn.testResult,
+                            ssn.testRunner,
+                            ssn.plugins,
+                            conn)
     res = ssn.hooks.startSubprocess(event)
     if event.handled and not res:
         conn.send(None)
@@ -308,7 +311,7 @@ def procserver(session_export, conn):
         # deal with the case that testid is something other
         # than a simple string.
         test = event.loader.loadTestsFromName(testid)
-        # xxx try/except?
+        # XXX If there a need to protect the loop? try/except?
         rlog.debug("Execute test %s (%s)", testid, test)
         executor(test, event.result)
         events = [e for e in ssn.hooks.flush()]
@@ -319,6 +322,38 @@ def procserver(session_export, conn):
     ssn.hooks.stopSubprocess(event)
 
 
+def import_session(rlog, session_export):
+    ssn = session.Session()
+    ssn.config = session_export['config']
+    ssn.hooks = RecordingPluginInterface()
+    ssn.verbosity = session_export['verbosity']
+    ssn.startDir = session_export['startDir']
+    ssn.topLevelDir = session_export['topLevelDir']
+    ssn.prepareSysPath()
+    loader_ = loader.PluggableTestLoader(ssn)
+    ssn.testLoader = loader_
+    result_ = result.PluggableTestResult(ssn)
+    ssn.testResult = result_
+    runner_ = runner.PluggableTestRunner(ssn)  # needed??
+    ssn.testRunner = runner_
+    # load and register plugins, forcing multiprocess to the end
+    ssn.plugins = [
+        plugin(session=ssn) for plugin in session_export['pluginClasses']
+        if plugin is not MultiProcess
+    ]
+    rlog.debug("Plugins loaded: %s", ssn.plugins)
+
+    for plugin in ssn.plugins:
+        plugin.register()
+        rlog.debug("Registered %s in subprocess", plugin)
+
+    # instantiating the plugin will register it.
+    ssn.plugins.append(MultiProcess(session=ssn))
+    rlog.debug("Registered %s in subprocess", MultiProcess)
+    ssn.plugins[-1].pluginsLoaded(events.PluginsLoadedEvent(ssn.plugins))
+    return ssn
+
+
 # test generator
 def gentests(conn):
     while True:


Test patch:
diff --git a/nose2/tests/functional/support/scenario/layers/test_layers.py b/nose2/tests/functional/support/scenario/layers/test_layers.py
index e800780..a35bb0f 100644
--- a/nose2/tests/functional/support/scenario/layers/test_layers.py
+++ b/nose2/tests/functional/support/scenario/layers/test_layers.py
@@ -180,6 +180,8 @@ class InnerD(unittest.TestCase):
     layer = LayerD
 
     def test(self):
+        """test with docstring
+        """
         self.assertEqual(
             {'base': 'setup',
              'layerD': 'setup'},
diff --git a/nose2/tests/functional/test_layers_plugin.py b/nose2/tests/functional/test_layers_plugin.py
index 9666dba..c658bbe 100644
--- a/nose2/tests/functional/test_layers_plugin.py
+++ b/nose2/tests/functional/test_layers_plugin.py
@@ -56,7 +56,7 @@ class TestLayers(FunctionalTestCase):
 Base
   test \(test_layers.Outer\) ... ok
   LayerD
-    test \(test_layers.InnerD\) ... ok
+    test \(test_layers.InnerD\) test with docstring ... ok
   LayerA
     test \(test_layers.InnerA\) ... ok
   LayerB
diff --git a/nose2/tests/unit/test_mp_plugin.py b/nose2/tests/unit/test_mp_plugin.py
index 02a9949..e808bdd 100644
--- a/nose2/tests/unit/test_mp_plugin.py
+++ b/nose2/tests/unit/test_mp_plugin.py
@@ -1,6 +1,7 @@
 from nose2 import session
 from nose2.tests._common import TestCase, Conn
 from nose2.plugins import mp
+from six.moves import configparser
 import sys
 
 
@@ -58,3 +59,19 @@ class TestMPPlugin(TestCase):
         finally:
             sys.platform = platform
 
+    def test_session_import(self):
+        config = configparser.ConfigParser()
+        config.add_section(mp.MultiProcess.configSection)
+        export_session = {
+            "config": config,
+            "verbosity": None,
+            "startDir": '',
+            "topLevelDir": '',
+            "pluginClasses": [mp.MultiProcess]
+        }
+        import logging
+        session = mp.import_session(logging.root, export_session)
+        self.assertIn('registerInSubprocess', session.hooks.methods)
+        self.assertIn('startSubprocess', session.hooks.methods)
+        self.assertIn('stopSubprocess', session.hooks.methods)
+        pass