Repository: tox-dev/tox (python)

Issue:
Running tox in --notest mode always succeeds
So the problem is that I run `tox --notest` in the `install` step in Travis CI and just `tox` in `script`.

Now, because there's a deps conflict it traced back and tox printed out an error. But the return code was `0` so travis just folded all output from `tox --notest` and it looked weirdly broken on the test stage. While the real reason is that it produced a pre-baked venv without crashing the whole thing in the right moment.

Current workaround is to manually parse the output:
https://github.com/ansible/molecule/blob/5f6bcc9/.travis.yml#L338-L341

Pull request description:
Correctly fail `tox --notest` when setup fails
Resolves #1097 

Patch:
diff --git a/docs/changelog/1095.bugfix.rst b/docs/changelog/1095.bugfix.rst
new file mode 100644
index 00000000..9d0abc24
--- /dev/null
+++ b/docs/changelog/1095.bugfix.rst
@@ -0,0 +1,1 @@
+Don't rewrite ``{posargs}`` substitution for absolute paths.
diff --git a/docs/changelog/1097.bugfix.rst b/docs/changelog/1097.bugfix.rst
new file mode 100644
index 00000000..e4936a94
--- /dev/null
+++ b/docs/changelog/1097.bugfix.rst
@@ -0,0 +1,1 @@
+Correctly fail ``tox --notest`` when setup fails.
diff --git a/src/tox/config.py b/src/tox/config.py
index b499a8aa..4d5e02be 100755
--- a/src/tox/config.py
+++ b/src/tox/config.py
@@ -184,10 +184,10 @@ class PosargsOption:
             if value:
                 args = []
                 for arg in config.option.args:
-                    if arg:
-                        origpath = config.invocationcwd.join(arg, abs=True)
-                        if origpath.check():
-                            arg = testenv_config.changedir.bestrelpath(origpath)
+                    if arg and not os.path.isabs(arg):
+                        origpath = os.path.join(config.invocationcwd.strpath, arg)
+                        if os.path.exists(origpath):
+                            arg = os.path.relpath(origpath, testenv_config.changedir.strpath)
                     args.append(arg)
             testenv_config._reader.addsubstitutions(args)
         return value
diff --git a/src/tox/session.py b/src/tox/session.py
index 92caedb3..b90a5332 100644
--- a/src/tox/session.py
+++ b/src/tox/session.py
@@ -608,7 +608,7 @@ class Session:
         envlog.set_installed(packages)
 
     def runtestenv(self, venv, redirect=False):
-        if self.config.option.notest:
+        if venv.status == 0 and self.config.option.notest:
             venv.status = "skipped tests"
         else:
             if venv.status:


Test patch:
diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py
index 86f36319..08a1ced4 100644
--- a/tests/unit/test_config.py
+++ b/tests/unit/test_config.py
@@ -11,6 +11,7 @@ import tox
 from tox.config import (
     CommandParser,
     DepOption,
+    PosargsOption,
     SectionReader,
     get_homedir,
     get_version_info,
@@ -2969,3 +2970,29 @@ def test_config_current_py(newconfig, current_tox_py, cmd, tmpdir, monkeypatch):
     assert config.envconfigs[current_tox_py]
     result = cmd()
     assert result.ret == 0, result.out
+
+
+def test_posargs_relative_changedir(newconfig, tmpdir):
+    dir1 = tmpdir.join("dir1").ensure()
+    tmpdir.join("dir2").ensure()
+    with tmpdir.as_cwd():
+        config = newconfig(
+            """\
+            [tox]
+            [testenv]
+            changedir = dir2
+            commands =
+                echo {posargs}
+            """
+        )
+        config.option.args = ["dir1", dir1.strpath, "dir3"]
+        testenv = config.envconfigs["python"]
+        PosargsOption().postprocess(testenv, config.option.args)
+
+        assert testenv._reader.posargs == [
+            # should have relative-ized
+            os.path.join("..", "dir1"),
+            # should have stayed the same,
+            dir1.strpath,
+            "dir3",
+        ]
diff --git a/tests/unit/test_z_cmdline.py b/tests/unit/test_z_cmdline.py
index 7bae2c22..98927f28 100644
--- a/tests/unit/test_z_cmdline.py
+++ b/tests/unit/test_z_cmdline.py
@@ -737,6 +737,22 @@ def test_notest(initproj, cmd):
     assert re.match(r".*py26\W+reusing.*", result.out, re.DOTALL)
 
 
+def test_notest_setup_py_error(initproj, cmd):
+    initproj(
+        "example123",
+        filedefs={
+            "setup.py": """\
+                from setuptools import setup
+                setup(name='x', install_requires=['fakefakefakefakefakefake']),
+            """,
+            "tox.ini": "",
+        },
+    )
+    result = cmd("--notest")
+    assert result.ret
+    assert re.search("ERROR:.*InvocationError", result.out)
+
+
 def test_PYC(initproj, cmd, monkeypatch):
     initproj("example123", filedefs={"tox.ini": ""})
     monkeypatch.setenv("PYTHONDOWNWRITEBYTECODE", 1)