Repository: confident-ai/deepeval
Issue: Custom LLM in the docs provide no schema input for generate method, and returns string, which causes summarization.py raise AttributeError: 'str' object has no attribute 'truths'
Root cause: Documentation Deficiency

`from deepeval.models.base_model import DeepEvalBaseLLM

class Mistral7B(DeepEvalBaseLLM):
    def __init__(
        self,
        model,
        tokenizer
    ):
        self.model = model
        self.tokenizer = tokenizer

    def load_model(self):
        return self.model

    def generate(self, prompt: str, schema=None) -> str:

        if not prompt or len(prompt.strip()) == 0:
            raise ValueError("Prompt cannot be empty")

        model = self.load_model()
        device = "cuda" 

        model_inputs = self.tokenizer([prompt], return_tensors="pt").to(device)
        model.to(device)

        if model_inputs.input_ids.numel() == 0:
            raise ValueError("Tokenized input is empty")

        generated_ids = model.generate(
            **model_inputs, 
            max_new_tokens=2000, 
            do_sample=True,
            pad_token_id=self.tokenizer.eos_token_id
            eos_token_id=self.tokenizer.eos_token_id
        )
        response = self.tokenizer.batch_decode(generated_ids)[0]
        print('response is...\n', response)
        
        return response

    async def a_generate(self, prompt: str, schema=None) -> str:
        return self.generate(prompt, schema)


    def get_model_name(self):
        return "Mistral 7B"`

Traceback (most recent call last):
  File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/gradio/queueing.py", line 625, in process_events
    response = await route_utils.call_process_api(
  File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/gradio/route_utils.py", line 322, in call_process_api
    output = await app.get_blocks().process_api(
  File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/gradio/blocks.py", line 2108, in process_api
    result = await self.call_function(
  File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/gradio/blocks.py", line 1655, in call_function
    prediction = await anyio.to_thread.run_sync(  # type: ignore
  File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/anyio/to_thread.py", line 56, in run_sync
    return await get_async_backend().run_sync_in_worker_thread(
  File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 2461, in run_sync_in_worker_thread
    return await future
  File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 962, in run
    result = context.run(func, *args)
  File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/gradio/utils.py", line 890, in wrapper
    response = f(*args, **kwargs)
  File "/home/user/app/app.py", line 66, in run_metric
    metric.measure(test_case)
  File "/home/user/app/deepeval/metrics/summarization/summarization.py", line 76, in measure
    loop.run_until_complete(
  File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
    return future.result()
  File "/home/user/app/deepeval/metrics/summarization/summarization.py", line 128, in a_measure
    self.truths, self.claims = await asyncio.gather(
  File "/home/user/app/deepeval/metrics/summarization/summarization.py", line 489, in _a_generate_truths
    return res.truths
AttributeError: 'str' object has no attribute 'truths'
 

@pin-lpt-heatherwick because you didn't return a pydantic `BaseModel` as is required when you define a `schema` argument. Either remove the schema argument or return a `BaseModel`.

I tried using the doc as is already as below

```
    def generate(self, prompt: str) -> str:
        model = self.load_model()

        device = "cuda" # the device to load the model onto

        model_inputs = self.tokenizer([prompt], return_tensors="pt").to(device)
        model.to(device)

        generated_ids = model.generate(**model_inputs, max_new_tokens=100, do_sample=True)
        return self.tokenizer.batch_decode(generated_ids)[0]

    async def a_generate(self, prompt: str) -> str:
        return self.generate(prompt)
```

which raises the error below

Traceback (most recent call last):
  File "/home/user/app/deepeval/metrics/summarization/summarization.py", line 488, in _a_generate_truths
    res: Truths = await self.model.a_generate(prompt, schema=Truths)
TypeError: Mistral7B.a_generate() got an unexpected keyword argument 'schema'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/user/app/deepeval/metrics/utils.py", line 258, in trimAndLoadJson
    return json.loads(jsonStr)
  File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/json/__init__.py", line 346, in loads
    return _default_decoder.decode(s)
  File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/json/decoder.py", line 337, in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
  File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/json/decoder.py", line 353, in raw_decode
    obj, end = self.scan_once(s, idx)
json.decoder.JSONDecodeError: Expecting ',' delimiter: line 4 column 9 (char 120)

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/gradio/queueing.py", line 625, in process_events
    response = await route_utils.call_process_api(
  File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/gradio/route_utils.py", line 322, in call_process_api
    output = await app.get_blocks().process_api(
  File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/gradio/blocks.py", line 2108, in process_api
    result = await self.call_function(
  File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/gradio/blocks.py", line 1655, in call_function
    prediction = await anyio.to_thread.run_sync(  # type: ignore
  File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/anyio/to_thread.py", line 56, in run_sync
    return await get_async_backend().run_sync_in_worker_thread(
  File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 2461, in run_sync_in_worker_thread
    return await future
  File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/anyio/_backends/_asyncio.py", line 962, in run
    result = context.run(func, *args)
  File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/site-packages/gradio/utils.py", line 890, in wrapper
    response = f(*args, **kwargs)
  File "/home/user/app/app.py", line 66, in run_metric
    metric.measure(test_case)
  File "/home/user/app/deepeval/metrics/summarization/summarization.py", line 76, in measure
    loop.run_until_complete(
  File "/home/user/.pyenv/versions/3.10.16/lib/python3.10/asyncio/base_events.py", line 649, in run_until_complete
    return future.result()
  File "/home/user/app/deepeval/metrics/summarization/summarization.py", line 128, in a_measure
    self.truths, self.claims = await asyncio.gather(
  File "/home/user/app/deepeval/metrics/summarization/summarization.py", line 492, in _a_generate_truths
    data = trimAndLoadJson(res, self)
  File "/home/user/app/deepeval/metrics/utils.py", line 263, in trimAndLoadJson
    raise ValueError(error_str)
ValueError: Evaluation LLM outputted an invalid JSON. Please use a better evaluation model.


@pin-lpt-heatherwick hmm i see, let me check back to you tomorrow on this metric, thanks for flagging it

hey @pin-lpt-heatherwick i had a look for you and it doesn't make sense because on line 488 which is where the `TypeError` occurred we did catch the type error exception and removed schema to be passed in: https://github.com/confident-ai/deepeval/blob/d1807b7d3d62c434157ffd04eb2fea2f2456e742/deepeval/metrics/summarization/summarization.py#L488

r u on the latest deepeval version?