Repository: getappmap/appmap-python (language: python)

Issue:
Report test_failure when a test fails
See appmap spec for details on this metadata field. 


Discussion / hints:

Fix patch:
diff --git a/.travis.yml b/.travis.yml
index 07a266a0..8b2547a9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -11,7 +11,10 @@ python:
 # https://github.com/travis-ci/travis-ci/issues/1147#issuecomment-441393807
 if: type != push OR branch = master OR branch =~ /^v\d+\.\d+(\.\d+)?(-\S*)?$/
 
-before_install: pip install --upgrade pip 'setuptools==65.6.2' 'poetry>=1.2.0'
+before_install: |
+  curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain stable --profile minimal
+  source "$HOME/.cargo/env"
+  pip install --upgrade pip 'setuptools==65.6.2' 'poetry>=1.2.0'
 
 install: pip install --upgrade "tox < 4" tox-travis
 script: tox


Test patch:
diff --git a/_appmap/test/data/pytest/expected/status_errored.metadata.json b/_appmap/test/data/pytest/expected/status_errored.metadata.json
deleted file mode 120000
index 35608576..00000000
--- a/_appmap/test/data/pytest/expected/status_errored.metadata.json
+++ /dev/null
@@ -1,1 +0,0 @@
-../../unittest/expected/status_errored.metadata.json
\ No newline at end of file
diff --git a/_appmap/test/data/pytest/expected/status_errored.metadata.json b/_appmap/test/data/pytest/expected/status_errored.metadata.json
new file mode 100644
index 00000000..45b3bed1
--- /dev/null
+++ b/_appmap/test/data/pytest/expected/status_errored.metadata.json
@@ -0,0 +1,11 @@
+{
+  "test_status": "failed",
+  "test_failure": {
+    "message": "RuntimeError: test error",
+    "location": "test_simple.py:28"
+  },
+  "exception": {
+    "class": "RuntimeError",
+    "message": "test error"
+  }
+}
diff --git a/_appmap/test/data/pytest/expected/status_failed.metadata.json b/_appmap/test/data/pytest/expected/status_failed.metadata.json
index 9c8a48e0..cc971c33 100644
--- a/_appmap/test/data/pytest/expected/status_failed.metadata.json
+++ b/_appmap/test/data/pytest/expected/status_failed.metadata.json
@@ -1,5 +1,9 @@
 {
   "test_status": "failed",
+  "test_failure": {
+    "message": "AssertionError: assert False",
+    "location": "test_simple.py:14"
+  },
   "exception": {
     "class": "AssertionError",
     "message": "assert False"
diff --git a/_appmap/test/data/pytest/expected/status_xfailed.metadata.json b/_appmap/test/data/pytest/expected/status_xfailed.metadata.json
deleted file mode 120000
index 8793d151..00000000
--- a/_appmap/test/data/pytest/expected/status_xfailed.metadata.json
+++ /dev/null
@@ -1,1 +0,0 @@
-status_failed.metadata.json
\ No newline at end of file
diff --git a/_appmap/test/data/pytest/expected/status_xfailed.metadata.json b/_appmap/test/data/pytest/expected/status_xfailed.metadata.json
new file mode 100644
index 00000000..992d824d
--- /dev/null
+++ b/_appmap/test/data/pytest/expected/status_xfailed.metadata.json
@@ -0,0 +1,11 @@
+{
+  "test_status": "failed",
+  "test_failure": {
+    "message": "AssertionError: assert False",
+    "location": "test_simple.py:19"
+  },
+  "exception": {
+    "class": "AssertionError",
+    "message": "assert False"
+  }
+}
diff --git a/_appmap/test/data/unittest/expected/status_errored.metadata.json b/_appmap/test/data/unittest/expected/status_errored.metadata.json
index f86990c5..f1e4a2e4 100644
--- a/_appmap/test/data/unittest/expected/status_errored.metadata.json
+++ b/_appmap/test/data/unittest/expected/status_errored.metadata.json
@@ -1,5 +1,9 @@
 {
   "test_status": "failed",
+  "test_failure": {
+    "message": "RuntimeError: test error",
+    "location": "simple/test_simple.py:34"
+  },
   "exception": {
     "class": "RuntimeError",
     "message": "test error"
diff --git a/_appmap/test/data/unittest/expected/status_failed.metadata.json b/_appmap/test/data/unittest/expected/status_failed.metadata.json
index c2bdf91a..93e98ae9 100644
--- a/_appmap/test/data/unittest/expected/status_failed.metadata.json
+++ b/_appmap/test/data/unittest/expected/status_failed.metadata.json
@@ -1,5 +1,9 @@
 {
   "test_status": "failed",
+  "test_failure": {
+    "message": "AssertionError: False is not true",
+    "location": "simple/test_simple.py:22"
+  },
   "exception": {
     "class": "AssertionError",
     "message": "False is not true"
diff --git a/_appmap/test/data/unittest/expected/status_xfailed.metadata.json b/_appmap/test/data/unittest/expected/status_xfailed.metadata.json
deleted file mode 120000
index 8793d151..00000000
--- a/_appmap/test/data/unittest/expected/status_xfailed.metadata.json
+++ /dev/null
@@ -1,1 +0,0 @@
-status_failed.metadata.json
\ No newline at end of file
diff --git a/_appmap/test/data/unittest/expected/status_xfailed.metadata.json b/_appmap/test/data/unittest/expected/status_xfailed.metadata.json
new file mode 100644
index 00000000..da5abda2
--- /dev/null
+++ b/_appmap/test/data/unittest/expected/status_xfailed.metadata.json
@@ -0,0 +1,11 @@
+{
+  "test_status": "failed",
+  "test_failure": {
+    "message": "AssertionError: False is not true",
+    "location": "simple/test_simple.py:26"
+  },
+  "exception": {
+    "class": "AssertionError",
+    "message": "False is not true"
+  }
+}
diff --git a/_appmap/testing_framework.py b/_appmap/testing_framework.py
index 217ec42b..cb11a918 100644
--- a/_appmap/testing_framework.py
+++ b/_appmap/testing_framework.py
@@ -1,15 +1,17 @@
 """Shared infrastructure for testing framework integration."""
 
 import os
+from pathlib import PurePath
 import re
 from contextlib import contextmanager
+import traceback
 
 import inflection
 
 from _appmap import configuration, env, generation, web_framework
 from _appmap.env import Env
 from _appmap.recording import Recording
-from _appmap.utils import fqname
+from _appmap.utils import fqname, root_relative_path
 
 from .metadata import Metadata
 
@@ -122,9 +124,7 @@ def record(self, klass, method, **kwds):
                 yield metadata
         finally:
             basedir = environ.output_dir / self.name
-            web_framework.write_appmap(
-                basedir, item.filename, generation.dump(rec, metadata)
-            )
+            web_framework.write_appmap(basedir, item.filename, generation.dump(rec, metadata))
 
 
 @contextmanager
@@ -137,6 +137,7 @@ def collect_result_metadata(metadata):
         yield
         metadata["test_status"] = "succeeded"
     except Exception as exn:
+        metadata["test_failure"] = {"message": failure_message(exn), "location": failure_location(exn)}
         metadata["test_status"] = "failed"
         metadata["exception"] = {"class": exn.__class__.__name__, "message": str(exn)}
         raise
@@ -147,3 +148,16 @@ def file_delete(filename):
         os.remove(filename)
     except FileNotFoundError:
         pass
+
+
+def failure_message(exn: Exception) -> str:
+    return f'{exn.__class__.__name__}: {exn}'
+
+
+def failure_location(exn: Exception) -> str:
+    # Exception could have been raised inside the test framework,
+    # but we want the location in the user code that caused it.
+    for frame in traceback.extract_tb(exn.__traceback__):
+        path = root_relative_path(frame.filename)
+        if not PurePath(path).is_absolute():
+            return f"{path}:{frame.lineno}"