Repository: promptfoo/promptfoo
Issue: Individual external Python assertion that raises exception results in no grading result in JSON output
Root cause: Validation Gap

If an individual Python assertion raises an exception in a test case, I would like to be able to see it as part of the `componentResults` in the JSON output. At the moment, it causes all of the grading results to be suppressed.

I'm using promptfoo@latest (0.60.0) and the example in `examples/python-assert-external`. 

First, I add another external assert file, `fails.py` with the following contents:

```python
def get_assert(output, context):
    return True
    #raise Exception("This is a forced failure")
```

Then, I modify `promptfooconfig.yaml` by using only one `vars` (yellow fruits) and adding `fails.py` as an additional assertion. So the file looks like this:

```yaml
prompts:
  - 'Tell me about {{topic}} in three words'
providers:
  - openai:gpt-4-1106-preview
tests:
  - vars:
      topic: yellow fruits
    assert:
      - type: python
        value: file://fails.py
      - type: python
        value: file://assert.py
```

Then, I run it:

`npx promptfoo@latest eval --config ./promptfooconfig.yaml --output okay.json`

<details><summary>okay.json output</summary>

```json
{
  "results": {
    "version": 2,
    "timestamp": "2024-05-28T18:51:25.690Z",
    "results": [
      {
        "provider": {
          "id": "openai:gpt-4-1106-preview"
        },
        "prompt": {
          "raw": "Tell me about yellow fruits in three words",
          "label": "Tell me about {{topic}} in three words"
        },
        "vars": {
          "topic": "yellow fruits"
        },
        "response": {
          "output": "Bright, nutritious, sweet.",
          "tokenUsage": {
            "cached": 21,
            "total": 21
          },
          "cached": true,
          "cost": 0.00033
        },
        "success": false,
        "score": 0.75,
        "namedScores": {},
        "latencyMs": 4,
        "cost": 0.00033,
        "error": "Contains banana",
        "gradingResult": {
          "pass": false,
          "score": 0.75,
          "reason": "Contains banana",
          "namedScores": {},
          "tokensUsed": {
            "total": 0,
            "prompt": 0,
            "completion": 0
          },
          "componentResults": [
            {
              "pass": true,
              "score": 1,
              "reason": "Assertion passed",
              "assertion": {
                "type": "python",
                "value": "file://fails.py"
              }
            },
            {
              "pass": false,
              "score": 0.5,
              "reason": "Contains banana",
              "assertion": {
                "type": "python",
                "value": "file://assert.py"
              }
            }
          ],
          "assertion": null
        }
      }
    ],
    "stats": {
      "successes": 0,
      "failures": 1,
      "tokenUsage": {
        "total": 21,
        "prompt": 0,
        "completion": 0,
        "cached": 21
      }
    },
    "table": {
      "head": {
        "prompts": [
          {
            "raw": "Tell me about {{topic}} in three words",
            "label": "Tell me about {{topic}} in three words",
            "id": "decba6b6c3d93395371f5593d5fc354ea253230714f97028b1da56e00119850f",
            "provider": "openai:gpt-4-1106-preview",
            "metrics": {
              "score": 0.75,
              "testPassCount": 0,
              "testFailCount": 1,
              "assertPassCount": 1,
              "assertFailCount": 1,
              "totalLatencyMs": 4,
              "tokenUsage": {
                "total": 21,
                "prompt": 0,
                "completion": 0,
                "cached": 21
              },
              "namedScores": {},
              "cost": 0.00033
            }
          }
        ],
        "vars": [
          "topic"
        ]
      },
      "body": [
        {
          "outputs": [
            {
              "pass": false,
              "score": 0.75,
              "namedScores": {},
              "text": "Contains banana\n---\nBright, nutritious, sweet.",
              "prompt": "Tell me about yellow fruits in three words",
              "provider": "openai:gpt-4-1106-preview",
              "latencyMs": 4,
              "tokenUsage": {
                "cached": 21,
                "total": 21
              },
              "gradingResult": {
                "pass": false,
                "score": 0.75,
                "reason": "Contains banana",
                "namedScores": {},
                "tokensUsed": {
                  "total": 0,
                  "prompt": 0,
                  "completion": 0
                },
                "componentResults": [
                  {
                    "pass": true,
                    "score": 1,
                    "reason": "Assertion passed",
                    "assertion": {
                      "type": "python",
                      "value": "file://fails.py"
                    }
                  },
                  {
                    "pass": false,
                    "score": 0.5,
                    "reason": "Contains banana",
                    "assertion": {
                      "type": "python",
                      "value": "file://assert.py"
                    }
                  }
                ],
                "assertion": null
              },
              "cost": 0.00033
            }
          ],
          "test": {
            "vars": {
              "topic": "yellow fruits"
            },
            "assert": [
              {
                "type": "python",
                "value": "file://fails.py"
              },
              {
                "type": "python",
                "value": "file://assert.py"
              }
            ],
            "options": {}
          },
          "vars": [
            "yellow fruits"
          ]
        }
      ]
    }
  },
  "config": {
    "prompts": [
      "Tell me about {{topic}} in three words"
    ],
    "providers": [
      "openai:gpt-4-1106-preview"
    ],
    "tests": [
      {
        "vars": {
          "topic": "yellow fruits"
        },
        "assert": [
          {
            "type": "python",
            "value": "file://fails.py"
          },
          {
            "type": "python",
            "value": "file://assert.py"
          }
        ]
      }
    ],
    "scenarios": [],
    "env": {},
    "sharing": true,
    "defaultTest": {
      "vars": {},
      "assert": [],
      "options": {}
    },
    "outputPath": [
      "okay.json"
    ]
  },
  "shareableUrl": null
}
```
</details> 

Now, if I change `fails.py` so that `get_assert` always raises an exception, and run the eval again:

`npx promptfoo@latest eval --config ./promptfooconfig.yaml --output exception.json`

Note how `exception.json` no longer has a `gradingResult` or `componentResult` dictionary.

<details><summary>exception.json</summary>

```json
{
  "results": {
    "version": 2,
    "timestamp": "2024-05-28T18:54:38.934Z",
    "results": [
      {
        "provider": {
          "id": "openai:gpt-4-1106-preview"
        },
        "prompt": {
          "raw": "Tell me about yellow fruits in three words",
          "label": "Tell me about {{topic}} in three words"
        },
        "vars": {
          "topic": "yellow fruits"
        },
        "error": "Error: Python script execution failed: Error: Error running Python script: Exception: This is a forced failure\nStack Trace: Error: Exception: This is a forced failure\n    at PythonShell.parseError (/Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/python-shell/index.js:303:21)\n    at terminateIfNeeded (/Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/python-shell/index.js:193:32)\n    at ChildProcess.<anonymous> (/Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/python-shell/index.js:185:13)\n    at ChildProcess.emit (node:events:519:28)\n    at ChildProcess._handle.onexit (node:internal/child_process:294:12)\n    --Python Traceback: --\n    File \"/Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/promptfoo/dist/src/python/wrapper.py\", line 27, in <module>\n      result = call_method(script_path, method_name, *data)\n               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n    File \"/Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/promptfoo/dist/src/python/wrapper.py\", line 18, in call_method\n      return method_to_call(*args)\n             ^^^^^^^^^^^^^^^^^^^^^\n    File \"/Users/efung/src/github.com/promptfoo/promptfoo/examples/python-assert-external/fails.py\", line 2, in get_assert\n      raise Exception(\"This is a forced failure\")\n\nError: Python script execution failed: Error: Error running Python script: Exception: This is a forced failure\nStack Trace: Error: Exception: This is a forced failure\n    at PythonShell.parseError (/Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/python-shell/index.js:303:21)\n    at terminateIfNeeded (/Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/python-shell/index.js:193:32)\n    at ChildProcess.<anonymous> (/Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/python-shell/index.js:185:13)\n    at ChildProcess.emit (node:events:519:28)\n    at ChildProcess._handle.onexit (node:internal/child_process:294:12)\n    --Python Traceback: --\n    File \"/Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/promptfoo/dist/src/python/wrapper.py\", line 27, in <module>\n      result = call_method(script_path, method_name, *data)\n               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n    File \"/Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/promptfoo/dist/src/python/wrapper.py\", line 18, in call_method\n      return method_to_call(*args)\n             ^^^^^^^^^^^^^^^^^^^^^\n    File \"/Users/efung/src/github.com/promptfoo/promptfoo/examples/python-assert-external/fails.py\", line 2, in get_assert\n      raise Exception(\"This is a forced failure\")\n    at runAssertion (/Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/promptfoo/dist/src/assertions.js:179:27)\n    at async /Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/promptfoo/dist/src/assertions.js:99:24",
        "success": false,
        "score": 0,
        "namedScores": {},
        "latencyMs": 3
      }
    ],
    "stats": {
      "successes": 0,
      "failures": 1,
      "tokenUsage": {
        "total": 0,
        "prompt": 0,
        "completion": 0,
        "cached": 0
      }
    },
    "table": {
      "head": {
        "prompts": [
          {
            "raw": "Tell me about {{topic}} in three words",
            "label": "Tell me about {{topic}} in three words",
            "id": "decba6b6c3d93395371f5593d5fc354ea253230714f97028b1da56e00119850f",
            "provider": "openai:gpt-4-1106-preview",
            "metrics": {
              "score": 0,
              "testPassCount": 0,
              "testFailCount": 1,
              "assertPassCount": 0,
              "assertFailCount": 0,
              "totalLatencyMs": 3,
              "tokenUsage": {
                "total": 0,
                "prompt": 0,
                "completion": 0,
                "cached": 0
              },
              "namedScores": {},
              "cost": 0
            }
          }
        ],
        "vars": [
          "topic"
        ]
      },
      "body": [
        {
          "outputs": [
            {
              "pass": false,
              "score": 0,
              "namedScores": {},
              "text": "Error: Python script execution failed: Error: Error running Python script: Exception: This is a forced failure\nStack Trace: Error: Exception: This is a forced failure\n    at PythonShell.parseError (/Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/python-shell/index.js:303:21)\n    at terminateIfNeeded (/Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/python-shell/index.js:193:32)\n    at ChildProcess.<anonymous> (/Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/python-shell/index.js:185:13)\n    at ChildProcess.emit (node:events:519:28)\n    at ChildProcess._handle.onexit (node:internal/child_process:294:12)\n    --Python Traceback: --\n    File \"/Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/promptfoo/dist/src/python/wrapper.py\", line 27, in <module>\n      result = call_method(script_path, method_name, *data)\n               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n    File \"/Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/promptfoo/dist/src/python/wrapper.py\", line 18, in call_method\n      return method_to_call(*args)\n             ^^^^^^^^^^^^^^^^^^^^^\n    File \"/Users/efung/src/github.com/promptfoo/promptfoo/examples/python-assert-external/fails.py\", line 2, in get_assert\n      raise Exception(\"This is a forced failure\")\n\nError: Python script execution failed: Error: Error running Python script: Exception: This is a forced failure\nStack Trace: Error: Exception: This is a forced failure\n    at PythonShell.parseError (/Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/python-shell/index.js:303:21)\n    at terminateIfNeeded (/Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/python-shell/index.js:193:32)\n    at ChildProcess.<anonymous> (/Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/python-shell/index.js:185:13)\n    at ChildProcess.emit (node:events:519:28)\n    at ChildProcess._handle.onexit (node:internal/child_process:294:12)\n    --Python Traceback: --\n    File \"/Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/promptfoo/dist/src/python/wrapper.py\", line 27, in <module>\n      result = call_method(script_path, method_name, *data)\n               ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n    File \"/Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/promptfoo/dist/src/python/wrapper.py\", line 18, in call_method\n      return method_to_call(*args)\n             ^^^^^^^^^^^^^^^^^^^^^\n    File \"/Users/efung/src/github.com/promptfoo/promptfoo/examples/python-assert-external/fails.py\", line 2, in get_assert\n      raise Exception(\"This is a forced failure\")\n    at runAssertion (/Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/promptfoo/dist/src/assertions.js:179:27)\n    at async /Users/efung/.npm/_npx/81bbc6515d992ace/node_modules/promptfoo/dist/src/assertions.js:99:24\n---\n",
              "prompt": "Tell me about yellow fruits in three words",
              "provider": "openai:gpt-4-1106-preview",
              "latencyMs": 3,
              "cost": 0
            }
          ],
          "test": {
            "vars": {
              "topic": "yellow fruits"
            },
            "assert": [
              {
                "type": "python",
                "value": "file://fails.py"
              },
              {
                "type": "python",
                "value": "file://assert.py"
              }
            ],
            "options": {}
          },
          "vars": [
            "yellow fruits"
          ]
        }
      ]
    }
  },
  "config": {
    "prompts": [
      "Tell me about {{topic}} in three words"
    ],
    "providers": [
      "openai:gpt-4-1106-preview"
    ],
    "tests": [
      {
        "vars": {
          "topic": "yellow fruits"
        },
        "assert": [
          {
            "type": "python",
            "value": "file://fails.py"
          },
          {
            "type": "python",
            "value": "file://assert.py"
          }
        ]
      }
    ],
    "scenarios": [],
    "env": {},
    "sharing": true,
    "defaultTest": {
      "vars": {},
      "assert": [],
      "options": {}
    },
    "outputPath": [
      "exception.json"
    ]
  },
  "shareableUrl": null
}
```
</details> 

Solved by #833.  

It will usually make sense for external scripts to handle their own errors, so they can provide useful errors.  This behavior is probably inconsistent across the multiple types of providers/integrations.