Repository: pytest-dev/execnet

Issue:
Tracebacks missing when using Gateway.remote_exec()
Hi.

If you call `remote_exec(mymodule)` and then an exception is raised, we get something like this

```
execnet.gateway_base.RemoteError: Traceback (most recent call last):
  File "<string>", line 1073, in executetask
  File "<remote exec>", line 79, in function1
  File "<remote exec>", line 77, in function2
ValueError: 'some error'
```
which is quite annoying to debug in a big application, as you have to hunt down where `remote_exec()` is called in the code, see which module was invoked, open the file and jump to the line manually.

The fix is simple. The proper file path just needs to be passed to the slave.


Discussion:

Patch:
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
index 99b85af..cebcf31 100644
--- a/CHANGELOG.rst
+++ b/CHANGELOG.rst
@@ -1,3 +1,9 @@
+1.6.3 (08-08-2019)
+------------------
+
+* `#102 <https://github.com/pytest-dev/execnet/pull/102>`__: Show paths in stack traces
+  generated by ``remote_exec()``.
+
 1.6.1 (2019-07-22)
 ------------------
 
diff --git a/execnet/gateway.py b/execnet/gateway.py
index 85421a1..e38e9ad 100644
--- a/execnet/gateway.py
+++ b/execnet/gateway.py
@@ -7,7 +7,6 @@ import sys
 import os
 import inspect
 import types
-import linecache
 import textwrap
 import execnet
 from execnet.gateway_base import Message
@@ -111,22 +110,27 @@ class Gateway(gateway_base.BaseGateway):
             executing code.
         """
         call_name = None
+        file_name = None
         if isinstance(source, types.ModuleType):
-            linecache.updatecache(inspect.getsourcefile(source))
-            source = inspect.getsource(source)
+            file_name = inspect.getsourcefile(source)
+            if not file_name:
+                source = inspect.getsource(source)
+            else:
+                source = None
         elif isinstance(source, types.FunctionType):
             call_name = source.__name__
+            file_name = inspect.getsourcefile(source)
             source = _source_of_function(source)
         else:
             source = textwrap.dedent(str(source))
 
-        if call_name is None and kwargs:
+        if not call_name and kwargs:
             raise TypeError("can't pass kwargs to non-function remote_exec")
 
         channel = self.newchannel()
         self._send(Message.CHANNEL_EXEC,
                    channel.id,
-                   gateway_base.dumps_internal((source, call_name, kwargs)))
+                   gateway_base.dumps_internal((source, file_name, call_name, kwargs)))
         return channel
 
     def remote_init_threads(self, num=None):
@@ -186,7 +190,7 @@ def _source_of_function(function):
         args = inspect.getargspec(function)[0]
     else:
         args = sig.args
-    if args[0] != 'channel':
+    if not args or args[0] != 'channel':
         raise ValueError('expected first function argument to be `channel`')
 
     if gateway_base.ISPY3:
@@ -213,4 +217,5 @@ def _source_of_function(function):
             used_globals,
         )
 
-    return source
+    leading_ws = "\n" * (codeobj.co_firstlineno - 1)
+    return leading_ws + source
diff --git a/execnet/gateway_base.py b/execnet/gateway_base.py
index 4f0a62d..2ddf5cd 100644
--- a/execnet/gateway_base.py
+++ b/execnet/gateway_base.py
@@ -13,6 +13,7 @@ NOTE: aims to be compatible to Python 2.5-3.X, Jython and IronPython
 """
 from __future__ import with_statement
 import sys
+import linecache
 import os
 import weakref
 import traceback
@@ -25,8 +26,9 @@ import struct
 ISPY3 = sys.version_info >= (3, 0)
 if ISPY3:
     from io import BytesIO
-    exec("def do_exec(co, loc): exec(co, loc)\n"
-         "def reraise(cls, val, tb): raise val\n")
+    exec("do_exec = exec")
+    def reraise(cls, val, tb):
+        raise val.with_traceback(tb)
     unicode = str
     _long_type = int
     from _thread import interrupt_main
@@ -1044,7 +1046,7 @@ class SlaveGateway(BaseGateway):
 
     def executetask(self, item):
         try:
-            channel, (source, call_name, kwargs) = item
+            channel, (source, file_name, call_name, kwargs) = item
             if not ISPY3 and kwargs:
                 # some python2 versions do not accept unicode keyword params
                 # note: Unserializer generally turns py2-str to py3-str objects
@@ -1054,12 +1056,15 @@ class SlaveGateway(BaseGateway):
                         name = name.encode('ascii')
                     newkwargs[name] = value
                 kwargs = newkwargs
+            if source is None:
+                assert file_name, file_name
+                source = "".join(linecache.updatecache(file_name))
             loc = {'channel': channel, '__name__': '__channelexec__'}
             self._trace("execution starts[%s]: %s" %
                         (channel.id, repr(source)[:50]))
             channel._executing = True
             try:
-                co = compile(source+'\n', '<remote exec>', 'exec')
+                co = compile(source+'\n', file_name or '<remote exec>', 'exec')
                 do_exec(co, loc)  # noqa
                 if call_name:
                     self._trace('calling %s(**%60r)' % (call_name, kwargs))
diff --git a/execnet/script/socketserver.py b/execnet/script/socketserver.py
index 36418bf..5117876 100755
--- a/execnet/script/socketserver.py
+++ b/execnet/script/socketserver.py
@@ -63,7 +63,7 @@ def exec_from_one_connection(serversock):
     }
     source = eval(source)
     if source:
-        co = compile(source+'\n', source, 'exec')
+        co = compile(source+'\n', '<socket server>', 'exec')
         print_(progname, 'compiled source, executing')
         try:
             exec_(co, g)  # noqa


Test patch:
diff --git a/testing/conftest.py b/testing/conftest.py
index 3630236..5aa2f08 100644
--- a/testing/conftest.py
+++ b/testing/conftest.py
@@ -186,8 +186,8 @@ def gw(request, execmodel, group):
 def execmodel(request):
     if request.param != "thread":
         pytest.importorskip(request.param)
-    if sys.platform == "win32":
-        pytest.xfail("eventlet/gevent do not work onwin32")
+    if request.param in ("eventlet", "gevent") and sys.platform == "win32":
+        pytest.xfail(request.param + " does not work on win32")
     return get_execmodel(request.param)
 
 
diff --git a/testing/test_basics.py b/testing/test_basics.py
index 76d7c03..e2598d7 100644
--- a/testing/test_basics.py
+++ b/testing/test_basics.py
@@ -218,8 +218,6 @@ def test_stdouterrin_setnull(execmodel):
     cap = py.io.StdCaptureFD()
     gateway_base.init_popen_io(execmodel)
     os.write(1, "hello".encode('ascii'))
-    if os.name == "nt":
-        os.write(2, "world")
     os.read(0, 1)
     out, err = cap.reset()
     assert not out
@@ -326,7 +324,7 @@ class TestSourceOfFunction(object):
         def working(channel):
             pass
 
-        send_source = gateway._source_of_function(working)
+        send_source = gateway._source_of_function(working).lstrip("\r\n")
         expected = 'def working(channel):\n    pass\n'
         assert send_source == expected
 
diff --git a/testing/test_gateway.py b/testing/test_gateway.py
index 4b82226..9de8bc7 100644
--- a/testing/test_gateway.py
+++ b/testing/test_gateway.py
@@ -2,6 +2,8 @@
 mostly functional tests of gateways.
 """
 import os
+from textwrap import dedent
+
 import py
 import pytest
 import socket
@@ -112,6 +114,38 @@ class TestBasicGateway:
         name = channel.receive()
         assert name == 2
 
+    def test_remote_exec_module_with_traceback(self, gw, tmpdir, monkeypatch):
+        remotetest = tmpdir.join("remotetest.py")
+        remotetest.write(dedent("""
+            def run_me(channel=None):
+                raise ValueError('me')
+
+            if __name__ == '__channelexec__':
+                run_me()
+            """)
+        )
+
+        monkeypatch.syspath_prepend(tmpdir)
+        import remotetest
+
+        ch = gw.remote_exec(remotetest)
+        try:
+            ch.receive()
+        except execnet.gateway_base.RemoteError as e:
+            assert 'remotetest.py", line 3, in run_me' in str(e)
+            assert "ValueError: me" in str(e)
+        finally:
+            ch.close()
+
+        ch = gw.remote_exec(remotetest.run_me)
+        try:
+            ch.receive()
+        except execnet.gateway_base.RemoteError as e:
+            assert 'remotetest.py", line 3, in run_me' in str(e)
+            assert "ValueError: me" in str(e)
+        finally:
+            ch.close()
+
     def test_correct_setup_no_py(self, gw):
         channel = gw.remote_exec("""
             import sys
diff --git a/testing/test_rsync.py b/testing/test_rsync.py
index 50d8db3..fb578ac 100644
--- a/testing/test_rsync.py
+++ b/testing/test_rsync.py
@@ -232,7 +232,7 @@ class TestRSync:
         assert len(dest.listdir()) == 1
         assert len(source.listdir()) == 1
 
-    @py.test.mark.skip_if('sys.version_info >= (3)')
+    @py.test.mark.skipif('sys.version_info >= (3,)')
     def test_2_to_3_bridge_can_send_binary_files(self, tmpdir, makegateway):
         python = _find_version('3')
         gw = makegateway('popen//python=%s' % python)