Repository: confident-ai/deepeval
Issue: error in command -> deepeval test run test_sample.py
Root cause: Environment Incompatibility

⠋ Downloading models (may take up to 2 minutes if running for the first time)...Exception in thread Thread-1:
Traceback (most recent call last):
  File "C:\Users\jayit\AppData\Local\Programs\Python\Python310\lib\threading.py", line 1009, in _bootstrap_inner
    self.run()
  File "C:\Users\jayit\deepeval\venv\lib\site-packages\rich\live.py", line 32, in run
    self.live.refresh()
  File "C:\Users\jayit\deepeval\venv\lib\site-packages\rich\live.py", line 241, in refresh
    with self.console:
  File "C:\Users\jayit\deepeval\venv\lib\site-packages\rich\console.py", line 864, in __exit__
    self._exit_buffer()
  File "C:\Users\jayit\deepeval\venv\lib\site-packages\rich\console.py", line 822, in _exit_buffer
    self._check_buffer()
  File "C:\Users\jayit\deepeval\venv\lib\site-packages\rich\console.py", line 2038, in _check_buffer
    write(text)
  File "C:\Users\jayit\AppData\Local\Programs\Python\Python310\lib\encodings\cp1252.py", line 19, in encode
    return codecs.charmap_encode(input,self.errors,encoding_table)[0]
UnicodeEncodeError: 'charmap' codec can't encode character '\u283c' in position 10: character maps to <undefined>
*** You may need to add PYTHONIOENCODING=utf-8 to your environment ***
FF.s                                                                                                                         [100%]
============================================================ FAILURES ============================================================= 
_____________________________________________________________ test_1 ______________________________________________________________ 

    def test_1():
        # Check to make sure it is relevant
        query = "What is the capital of France?"
        output = "The capital of France is Paris."
        metric = RandomMetric()
        # Comment this out for differne metrics/models
        # metric = AnswerRelevancyMetric(minimum_score=0.5)
        test_case = LLMTestCase(query=query, output=output)
>       assert_test(test_case, [metric])

test_sample.py:18:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
venv\lib\site-packages\deepeval\run_test.py:252: in assert_test
    return run_test(
venv\lib\site-packages\deepeval\run_test.py:239: in run_test
    measure_metric()
venv\lib\site-packages\deepeval\retry.py:39: in wrapper
    raise last_error  # Raise the last error
venv\lib\site-packages\deepeval\retry.py:23: in wrapper
    result = func(*args, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

    @retry(
        max_retries=max_retries, delay=delay, min_success=min_success
    )
    def measure_metric():
        score = metric.measure(test_case)
        success = metric.is_successful()
        if isinstance(test_case, LLMTestCase):
            log(
                success=success,
                score=score,
                metric=metric,
                query=test_case.query if test_case.query else "-",
                output=test_case.output if test_case.output else "-",
                expected_output=test_case.expected_output
                if test_case.expected_output
                else "-",
                context=test_case.context if test_case.context else "-",
            )

            test_result = TestResult(
                success=success,
                score=score,
                metric_name=metric.__name__,
                query=test_case.query if test_case.query else "-",
                output=test_case.output if test_case.output else "-",
                expected_output=test_case.expected_output
                if test_case.expected_output
                else "-",
                metadata=None,
                context=test_case.context,
            )
        elif isinstance(test_case, SearchTestCase):
            log(
                success=success,
                score=score,
                metric=metric,
                query=test_case.query if test_case.query else "-",
                output=str(test_case.output_list)
                if test_case.output_list
                else "-",
                expected_output=str(test_case.golden_list)
                if test_case.golden_list
                else "-",
                context="-",
            )
            test_result = TestResult(
                success=success,
                score=score,
                metric_name=metric.__name__,
                query=test_case.query if test_case.query else "-",
                output=test_case.output_list
                if test_case.output_list
                else "-",
                expected_output=test_case.golden_list
                if test_case.golden_list
                else "-",
                metadata=None,
                context="-",
            )
        else:
            raise ValueError("TestCase not supported yet.")
        test_results.append(test_result)

        if raise_error:
>           assert (
                metric.is_successful()
            ), f"{metric.__name__} failed. Score: {score}."
E           AssertionError: Random failed. Score: 0.2304173247532807.

venv\lib\site-packages\deepeval\run_test.py:235: AssertionError
------------------------------------------------------ Captured stdout call ------------------------------------------------------- 
Attempt 1 failed: Random failed. Score: 0.2304173247532807.
Max retries (1) exceeded.
_____________________________________________________________ test_2 ______________________________________________________________ 

    def test_2():
        # Check to make sure it is factually consistent
        output = "Cells have many major components, including the cell membrane, nucleus, mitochondria, and endoplasmic reticulum." 
        context = "Biology"
        metric = RandomMetric()
        # Comment this out for factual consistency tests
        # metric = FactualConsistencyMetric(minimum_score=0.8)
        test_case = LLMTestCase(output=output, context=context)
>       assert_test(test_case, [metric])

test_sample.py:29:
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
venv\lib\site-packages\deepeval\run_test.py:252: in assert_test
    return run_test(
venv\lib\site-packages\deepeval\run_test.py:239: in run_test
    measure_metric()
venv\lib\site-packages\deepeval\retry.py:39: in wrapper
    raise last_error  # Raise the last error
venv\lib\site-packages\deepeval\retry.py:23: in wrapper
    result = func(*args, **kwargs)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

    @retry(
        max_retries=max_retries, delay=delay, min_success=min_success
    )
    def measure_metric():
        score = metric.measure(test_case)
        success = metric.is_successful()
        if isinstance(test_case, LLMTestCase):
            log(
                success=success,
                score=score,
                metric=metric,
                query=test_case.query if test_case.query else "-",
                output=test_case.output if test_case.output else "-",
                expected_output=test_case.expected_output
                if test_case.expected_output
                else "-",
                context=test_case.context if test_case.context else "-",
            )

            test_result = TestResult(
                success=success,
                score=score,
                metric_name=metric.__name__,
                query=test_case.query if test_case.query else "-",
                output=test_case.output if test_case.output else "-",
                expected_output=test_case.expected_output
                if test_case.expected_output
                else "-",
                metadata=None,
                context=test_case.context,
            )
        elif isinstance(test_case, SearchTestCase):
            log(
                success=success,
                score=score,
                metric=metric,
                query=test_case.query if test_case.query else "-",
                output=str(test_case.output_list)
                if test_case.output_list
                else "-",
                expected_output=str(test_case.golden_list)
                if test_case.golden_list
                else "-",
                context="-",
            )
            test_result = TestResult(
                success=success,
                score=score,
                metric_name=metric.__name__,
                query=test_case.query if test_case.query else "-",
                output=test_case.output_list
                if test_case.output_list
                else "-",
                expected_output=test_case.golden_list
                if test_case.golden_list
                else "-",
                metadata=None,
                context="-",
            )
        else:
            raise ValueError("TestCase not supported yet.")
        test_results.append(test_result)

        if raise_error:
>           assert (
                metric.is_successful()
            ), f"{metric.__name__} failed. Score: {score}."
E           AssertionError: Random failed. Score: 0.2175566574653277.

venv\lib\site-packages\deepeval\run_test.py:235: AssertionError
------------------------------------------------------ Captured stdout call ------------------------------------------------------- 
Attempt 1 failed: Random failed. Score: 0.2175566574653277.
Max retries (1) exceeded.
====================================================== slowest 10 durations ======================================================= 
4.99s call     test_sample.py::test_1
2.37s call     test_sample.py::test_2
2.32s call     test_sample.py::test_3

(7 durations < 0.005s hidden.  Use -vv to show these durations.)
===================================================== short test summary info ===================================================== 
FAILED test_sample.py::test_1 - AssertionError: Random failed. Score: 0.2304173247532807.
FAILED test_sample.py::test_2 - AssertionError: Random failed. Score: 0.2175566574653277.
2 failed, 1 passed, 1 skipped in 10.94s
✅ Tests finished! View results on https://app.confident-ai.com/