Repository: explodinggradients/ragas
Issue: Automatic language adaptation Bugs. Cannot do with german language
Root cause: Validation Gap

[ ] I have checked the [documentation](https://docs.ragas.io/) and related resources and couldn't resolve my bug.

**Describe the bug**
A clear and concise description of what the bug is.

Ragas version: 0.1.20
Python version: Python 3.12.6

**Code to Reproduce**
from langchain_core.language_models import BaseLanguageModel
from langchain_core.outputs.llm_result import LLMResult
from langchain_core.outputs.generation import Generation
from langchain_core.runnables.config import run_in_executor
from langchain_core.embeddings import Embeddings
from langchain_core.messages import BaseMessage
from langchain_community.document_loaders import DirectoryLoader
from langchain_community.embeddings import HuggingFaceEmbeddings
from ragas.embeddings import LangchainEmbeddingsWrapper
from ragas.testset.generator import TestsetGenerator
from ragas.llms.prompt import PromptValue
from ragas.testset.evolutions import simple, reasoning,conditional
from typing import List, Optional, Any, Sequence
import requests
import json
import traceback
import logging

logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

    
def handle_response(response_text):
    # Split the response into lines
    lines = response_text.strip().split('\n')

    # Initialize variables to store the accumulated response
    accumulated_response = []
    is_done = False

    for line in lines:
        try:
            # Parse the JSON object from the line
            response_obj = json.loads(line)
        except json.JSONDecodeError:
            continue  # Skip lines that are not valid JSON

        # Append the response piece to the accumulated list
        if not is_done:
            accumulated_response.append(response_obj.get("response", ""))

        # Check if the response is marked as done
        if response_obj.get("done", False):
            is_done = True
            # Append the final piece to the accumulated response
            accumulated_response.append(response_obj.get("response", ""))
            break

    # Join all accumulated response pieces into a single string
    full_response = ''.join(accumulated_response).strip()
    
    return full_response

class OllamaLLM(BaseLanguageModel):
    model: str = "llama3"
    
    def generate_prompt(
        self,
        prompts: List[PromptValue],
        stop: Optional[List[str]] = None,
        callbacks: Optional[List[callable]] = None,
        **kwargs,
    ) -> LLMResult:
        url = "http://localhost:11434/api/generate"
        print("run normal generate_prompt")
        try:
            prompt = prompts[0].to_string()
            print("Prompt------------------------------")
            print(prompt)

            response = requests.post(url, json={"model": self.model, "prompt": prompt}, timeout=120,stream=False)
            response.raise_for_status()

            model_response = handle_response(response.text)
            print("Response------------------------------")
            print(model_response)
            res = Generation(text=model_response)
            

            return LLMResult(generations=[[res]])
        except requests.exceptions.RequestException as e:
            print(f"Error making request to Ollama API: {e}")
            return ""

    async def agenerate_prompt(
        self,
        prompts: List[PromptValue],
        stop: Optional[List[str]] = None,
        callbacks: Optional[List[callable]] = None,
        **kwargs,
    ) -> LLMResult:
        print("run agenerate_prompt")
        results = self.generate_prompt(prompts, stop=stop, **kwargs)
        print("FINISH agenerate_prompt")
        return results
        #return LLMResult(generations=results)
    

    def predict(
        self, text: str, *, stop: Optional[Sequence[str]] = None, **kwargs: Any
    ) -> str:
        return ""
    
    def predict_messages(
        self,
        messages: List[BaseMessage],
        *,
        stop: Optional[Sequence[str]] = None,
        **kwargs: Any,
    ) -> BaseMessage:
        return None
    async def apredict(
        self, text: str, *, stop: Optional[Sequence[str]] = None, **kwargs: Any
    ) -> str:
        return ""
    
    async def apredict_messages(
        self,
        messages: List[BaseMessage],
        *,
        stop: Optional[Sequence[str]] = None,
        **kwargs: Any,
    ) -> BaseMessage:
        return None
    def invoke():
        return None

class EmbeddingOllama(Embeddings):

    url = "http://localhost:11434" + "/api/embeddings"
    def __init__(self, model = 'bge-m3'): # default
        self.model = model
    def embed_documents(self, texts: List[str]) -> List[List[float]]:
        res : List[str] = []
        for i, text in enumerate(texts):
            try:
                response = requests.post(self.url, json={"model": self.model, "prompt": text})
                response.raise_for_status()
                response_json = response.json()

                res.append(response_json["embedding"])
            except requests.exceptions.HTTPError as http_err:
                print(f"HTTP error occurred: {http_err}")  # Handle specific HTTP error
            except requests.exceptions.ConnectionError as conn_err:
                print(f"Connection error occurred: {conn_err}")  # Handle connection issues
            except requests.exceptions.Timeout as timeout_err:
                print(f"Timeout error occurred: {timeout_err}")  # Handle timeout errors
            except requests.exceptions.RequestException as req_err:
                print(f"An error occurred: {req_err}")
        return res
          

    def embed_query(self, text: str) -> List[float]:
        try:
            response = requests.post(self.url, json={"model": self.model, "prompt": text})
            response.raise_for_status()
            response_json = response.json()
            return response_json["embedding"]
        except requests.exceptions.HTTPError as http_err:
            print(f"HTTP error occurred: {http_err}")  # Handle specific HTTP error
        except requests.exceptions.ConnectionError as conn_err:
            print(f"Connection error occurred: {conn_err}")  # Handle connection issues
        except requests.exceptions.Timeout as timeout_err:
            print(f"Timeout error occurred: {timeout_err}")  # Handle timeout errors
        except requests.exceptions.RequestException as req_err:
            print(f"An error occurred: {req_err}")  
        

    async def aembed_documents(self, texts: List[str]) -> List[List[float]]:
        return await run_in_executor(None, self.embed_documents, texts)

    async def aembed_query(self, text: str) -> List[float]:
        return await run_in_executor(None, self.embed_query, text)

def main():
    model_name = "llama3.1"
    # embedding_model = "bge-m3"
    langchain_llm = OllamaLLM(model=model_name)
    langchain_embeddings = HuggingFaceEmbeddings(model_name="BAAI/bge-m3")
    #langchain_embeddings = EmbeddingOllama(model=embedding_model)
    wrapped_embeddings = LangchainEmbeddingsWrapper(langchain_embeddings)
    
    try:
        loader = DirectoryLoader("./data")
        documents = loader.load()
        for document in documents:
            document.metadata['filename'] = document.metadata['source']
        generator = TestsetGenerator.from_langchain(
            langchain_llm,  # generator LLM
            langchain_llm,  # critic LLM
            wrapped_embeddings
        )
        try:
            language = "deutsch"
            generator.adapt(language, evolutions=[simple, reasoning,conditional])
            
            distributions = {
                simple: 0.4,
                reasoning: 0.3,
                conditional: 0.3
            }
            testset = generator.generate_with_langchain_docs(
                documents, 
                20, 
                distributions,
                with_debugging_logs=True, 
            )
            df = testset.to_pandas()
            df.to_excel("generate_de_results.xlsx", index=False)
        except Exception as e:
            print(f"Error in generate_with_langchain_docs: {e}")
            print(traceback.format_exc())
    except Exception as e:
        print(f"An error occurred: {e}")

if __name__ == "__main__":
    main()

**Error trace**
Error in generate_with_langchain_docs: invalid syntax (<unknown>, line 1)
Traceback (most recent call last):
  File "/Users/nguyenthanhdat/Documents/Projekte/test/generate_ragas.py", line 195, in main
    generator.adapt(language, evolutions=[simple, reasoning,conditional])
  File "/Users/nguyenthanhdat/Documents/Projekte/test/myenv/lib/python3.12/site-packages/ragas/testset/generator.py", line 340, in adapt
    self.docstore.extractor.adapt(language, cache_dir=cache_dir)
  File "/Users/nguyenthanhdat/Documents/Projekte/test/myenv/lib/python3.12/site-packages/ragas/testset/extractor.py", line 61, in adapt
    self.extractor_prompt = self.extractor_prompt.adapt(
                            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/Users/nguyenthanhdat/Documents/Projekte/test/myenv/lib/python3.12/site-packages/ragas/llms/prompt.py", line 246, in adapt
    example_dict[self.output_key] = ast.literal_eval(dict_str)
                                    ^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/homebrew/Cellar/python@3.12/3.12.6/Frameworks/Python.framework/Versions/3.12/lib/python3.12/ast.py", line 66, in literal_eval
    node_or_string = parse(node_or_string.lstrip(" \t"), mode='eval')
                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/opt/homebrew/Cellar/python@3.12/3.12.6/Frameworks/Python.framework/Versions/3.12/lib/python3.12/ast.py", line 52, in parse
    return compile(source, filename, mode, flags,
           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "<unknown>", line 1
    To translate the values in the given JSON to the target language
       ^^^^^^^^^
SyntaxError: invalid syntax

**Expected behavior**
I except RAGAS can generate german dataset for me.


**Additional context**
Here I use LLM from Ollama (LLama3.1) and HuggingFace Embedding

<!-- PS: bugs suck but is also part of the process. We sincerely apologies for breaking your flow because of it, but don't worry, we got your back ❤️. We will get this fixed as fast as we can and thanks for helping us out by reporting it 🙏. -->


<!-- Greeting -->
Hey there, @DatTNH23! I'm here to help you with your bug and answer any questions you may have. Let's tackle this together!

<!-- Answer -->
I found a similar open issue: [Generate test data for 1 pdf](https://github.com/explodinggradients/ragas/issues/1325), which is still open and was last updated on September 20, 2024. This issue was referenced by another issue: [Runner thread raised exception and testset function is not accessible](https://github.com/explodinggradients/ragas/issues/1333), last updated on September 24, 2024.

To resolve the syntax error in the `generate_with_langchain_docs` method when using RAGAS to generate a German dataset with Ollama LLM and HuggingFace Embedding, ensure that the method is properly indented and correctly formatted. Here is the corrected version of the `generate_with_langchain_docs` method:

```python
def generate_with_langchain_docs(
    self,
    docs: t.Sequence[Document],
    test_size: int,
    distribution: QADistribution,
) -> TestDataset:
    return self.generate(docs, test_size, distribution)
```

Make sure that the `return` statement is correctly indented and placed within the method. This should resolve the syntax error <sup>[[1]](https://github.com/explodinggradients/ragas/blob/main/src/ragas/testset/generator.py)</sup><sup>[[2]](https://github.com/explodinggradients/ragas/blob/main/src/experimental/ragas_experimental/testset/generators/base.py)</sup>.

Additionally, ensure that the `language` parameter is correctly set and used in the `adapt` method. The `language` parameter must be a string and is used to adapt various components to ensure they function correctly in the specified language <sup>[[1]](https://github.com/explodinggradients/ragas/blob/main/src/ragas/testset/generator.py)</sup><sup>[[3]](https://github.com/explodinggradients/ragas/blob/main/src/ragas/testset/evolutions.py)</sup>.

If you continue to face issues, please check the implementation of the `adapt` method in the `ragas/testset/extractor.py` file:

```python
def adapt(self, language: str, cache_dir: t.Optional[str] = None) -> None:
    """
    Adapt the extractor to a different language.
    """
    self.extractor_prompt = self.extractor_prompt.adapt(
        language, self.llm, cache_dir
    )
```

This method adapts the extractor to the specified language <sup>[[4]](https://github.com/explodinggradients/ragas/blob/main/src/ragas/testset/extractor.py)</sup>.

Lastly, ensure that the `extractor_prompt.adapt` method in the `ragas/llms/prompt.py` file correctly handles the adaptation process:

```python
example_dict = {}
example_dict.update(
    {k: v for k, v in zip(self.input_keys, example[: len(self.input_keys)])}
)
if self.output_type.lower() == "json":
    example_dict[self.output_key] = json_loader._safe_load(example[-1], llm)
    if example_dict[self.output_key] == {}:
        dict_str = example[-1].split("(")[0].strip()
        example_dict[self.output_key] = ast.literal_eval(dict_str)
    else:
        example_dict[self.output_key] = example[-1]
if self.output_type.lower() == "json":
    output = example_dict[self.output_key]
    if isinstance(output, dict):
        assert (
            set(output.keys()) == output_keys[i]
        ), f"Adapted output keys {set(output.keys())=} do not match with the original output keys: {output_keys[i]=}"
    elif isinstance(output, list) and all(
        isinstance(item, dict) for item in output
    ):
        assert all(
            set(item.keys()) in output_keys[i] for item in output
        ), "Adapted output keys do not match with the original output keys"
```

This ensures that the `example_dict` is correctly populated and validated <sup>[[5]](https://github.com/explodinggradients/ragas/blob/main/src/ragas/llms/prompt.py)</sup>.


<!-- Dosu Comment Footer -->
*To continue talking to [Dosu](https://dosu.dev), mention @dosu.*


@dosu It doesn't help, those 2 Issues are not relevant to my issue

<!-- Answer -->
To resolve the syntax error in the `generate_with_langchain_docs` method when using RAGAS to generate a German dataset with Ollama LLM and HuggingFace Embedding, ensure that the method is properly indented and correctly formatted. Here is the corrected version of the `generate_with_langchain_docs` method:

```python
def generate_with_langchain_docs(
    self,
    docs: t.Sequence[Document],
    test_size: int,
    distribution: QADistribution,
) -> TestDataset:
    return self.generate(docs, test_size, distribution)
```

Make sure that the `return` statement is correctly indented and placed within the method. This should resolve the syntax error <sup>[[1]](https://github.com/explodinggradients/ragas/blob/main/src/ragas/testset/generator.py)</sup><sup>[[2]](https://github.com/explodinggradients/ragas/blob/main/src/experimental/ragas_experimental/testset/generators/base.py)</sup>.

Additionally, ensure that the `language` parameter is correctly set and used in the `adapt` method. The `language` parameter must be a string and is used to adapt various components to ensure they function correctly in the specified language <sup>[[1]](https://github.com/explodinggradients/ragas/blob/main/src/ragas/testset/generator.py)</sup>.

If you continue to face issues, please check the implementation of the `adapt` method in the `ragas/testset/extractor.py` file:

```python
def adapt(self, language: str, cache_dir: t.Optional[str] = None) -> None:
    """
    Adapt the extractor to a different language.
    """
    self.extractor_prompt = self.extractor_prompt.adapt(
        language, self.llm, cache_dir
    )
```

This method adapts the extractor to the specified language <sup>[[3]](https://github.com/explodinggradients/ragas/blob/main/src/ragas/testset/extractor.py)</sup>.

Lastly, ensure that the `extractor_prompt.adapt` method in the `ragas/llms/prompt.py` file correctly handles the adaptation process:

```python
example_dict = {}
example_dict.update(
    {k: v for k, v in zip(self.input_keys, example[: len(self.input_keys)])}
)
if self.output_type.lower() == "json":
    example_dict[self.output_key] = json_loader._safe_load(example[-1], llm)
    if example_dict[self.output_key] == {}:
        dict_str = example[-1].split("(")[0].strip()
        example_dict[self.output_key] = ast.literal_eval(dict_str)
    else:
        example_dict[self.output_key] = example[-1]
if self.output_type.lower() == "json":
    output = example_dict[self.output_key]
    if isinstance(output, dict):
        assert (
            set(output.keys()) == output_keys[i]
        ), f"Adapted output keys {set(output.keys())=} do not match with the original output keys: {output_keys[i]=}"
    elif isinstance(output, list) and all(
        isinstance(item, dict) for item in output
    ):
        assert all(
            set(item.keys()) in output_keys[i] for item in output
        ), "Adapted output keys do not match with the original output keys"
```

This ensures that the `example_dict` is correctly populated and validated <sup>[[4]](https://github.com/explodinggradients/ragas/blob/main/src/ragas/llms/prompt.py)</sup>.


<!-- Dosu Comment Footer -->
*To continue talking to [Dosu](https://dosu.dev), mention @dosu.*