Repository: explodinggradients/ragas
Issue: No persona found during test set generation in non-english language
Root cause: Validation Gap

[x] I have checked the [documentation](https://docs.ragas.io/) and related resources and couldn't resolve my bug.

**Describe the bug**
No persona found with name ... during test set generation in non english language

Ragas version: 2.6
Python version: 3.10.6

**Code to Reproduce**
```py
from llama_index.core import SimpleDirectoryReader
from llama_index.embeddings.openai import OpenAIEmbedding
from llama_index.llms.openai import OpenAI
from ragas.llms import LlamaIndexLLMWrapper
from ragas.testset import TestsetGenerator
from ragas.testset.synthesizers.single_hop.specific import (
	SingleHopSpecificQuerySynthesizer,
)
from siemens_rag.utils import get_raw_path

generator_llm = OpenAI()
embeddings = OpenAIEmbedding(model="text-embedding-3-large")

input_path = get_raw_path()
documents = SimpleDirectoryReader(input_path, recursive=False).load_data()

ragas_llm = LlamaIndexLLMWrapper(generator_llm)

query_distribution = [
	(SingleHopSpecificQuerySynthesizer(llm=ragas_llm), 1.0),
]
for query, _ in query_distribution:
	prompts = await query.adapt_prompts("deutsch", llm=ragas_llm)
	query.set_prompts(**prompts)

generator = TestsetGenerator.from_llama_index(
	llm=generator_llm,
	embedding_model=embeddings,
)

testset = generator.generate_with_llamaindex_docs(
	documents,
	query_distribution=query_distribution,
	testset_size=2,
)

```

**Error trace**
```
KeyError                                  Traceback (most recent call last)
Cell In[36], line 1
----> 1 testset = generator.generate_with_llamaindex_docs(
      2     documents,
      3     query_distribution=query_distribution,
      4     testset_size=2,
      5 
      6 
      7 )

File ~/Library/Caches/pypoetry/virtualenvs/siemens-rag-65vf45KI-py3.10/lib/python3.10/site-packages/ragas/testset/synth
esizers/generate.py:265, in TestsetGenerator.generate_with_llamaindex_docs(self, documents, testset_size, transforms, t
ransforms_llm, transforms_embedding_model, query_distribution, run_config, callbacks, with_debugging_logs, raise_except
ions)
    262 apply_transforms(kg, transforms, run_config)
    263 self.knowledge_graph = kg
--> 265 return self.generate(
    266     testset_size=testset_size,
    267     query_distribution=query_distribution,
    268     run_config=run_config,
    269     callbacks=callbacks,
    270     with_debugging_logs=with_debugging_logs,
    271     raise_exceptions=raise_exceptions,
    272 )

File ~/Library/Caches/pypoetry/virtualenvs/siemens-rag-65vf45KI-py3.10/lib/python3.10/site-packages/ragas/testset/synth
esizers/generate.py:413, in TestsetGenerator.generate(self, testset_size, query_distribution, num_personas, run_config,
 batch_size, callbacks, token_usage_parser, with_debugging_logs, raise_exceptions)
    411 except Exception as e:
    412     scenario_generation_rm.on_chain_error(e)
--> 413     raise e
    414 else:
    415     scenario_generation_rm.on_chain_end(
    416         outputs={"scenario_sample_list": scenario_sample_list}
    417     )

File ~/Library/Caches/pypoetry/virtualenvs/siemens-rag-65vf45KI-py3.10/lib/python3.10/site-packages/ragas/testset/synth
esizers/generate.py:410, in TestsetGenerator.generate(self, testset_size, query_distribution, num_personas, run_config,
 batch_size, callbacks, token_usage_parser, with_debugging_logs, raise_exceptions)
    401     exec.submit(
    402         scenario.generate_scenarios,
    403         n=splits[i],
   (...)
    406         callbacks=scenario_generation_grp,
    407     )
    409 try:
--> 410     scenario_sample_list: t.List[t.List[BaseScenario]] = exec.results()
    411 except Exception as e:
    412     scenario_generation_rm.on_chain_error(e)

File ~/Library/Caches/pypoetry/virtualenvs/siemens-rag-65vf45KI-py3.10/lib/python3.10/site-packages/ragas/executor.py:2
13, in Executor.results(self)
    210             nest_asyncio.apply()
    211             self._nest_asyncio_applied = True
--> 213 results = asyncio.run(self._process_jobs())
    214 sorted_results = sorted(results, key=lambda x: x[0])
    215 return [r[1] for r in sorted_results]

File ~/Library/Caches/pypoetry/virtualenvs/siemens-rag-65vf45KI-py3.10/lib/python3.10/site-packages/nest_asyncio.py:30,
 in _patch_asyncio.<locals>.run(main, debug)
     28 task = asyncio.ensure_future(main)
     29 try:
---> 30     return loop.run_until_complete(task)
     31 finally:
     32     if not task.done():

File ~/Library/Caches/pypoetry/virtualenvs/siemens-rag-65vf45KI-py3.10/lib/python3.10/site-packages/nest_asyncio.py:98,
 in _patch_loop.<locals>.run_until_complete(self, future)
     95 if not f.done():
     96     raise RuntimeError(
     97         'Event loop stopped before Future completed.')
---> 98 return f.result()

File ~/.pyenv/versions/3.10.6/lib/python3.10/asyncio/futures.py:201, in Future.result(self)
    199 self.__log_traceback = False
    200 if self._exception is not None:
--> 201     raise self._exception.with_traceback(self._exception_tb)
    202 return self._result

File ~/.pyenv/versions/3.10.6/lib/python3.10/asyncio/tasks.py:232, in Task.__step(***failed resolving arguments***)
    228 try:
    229     if exc is None:
    230         # We use the `send` method directly, because coroutines
    231         # don't have `__iter__` and `__next__` methods.
--> 232         result = coro.send(None)
    233     else:
    234         result = coro.throw(exc)

File ~/Library/Caches/pypoetry/virtualenvs/siemens-rag-65vf45KI-py3.10/lib/python3.10/site-packages/ragas/executor.py:1
41, in Executor._process_jobs(self)
    135 if self.pbar is None:
    136     with tqdm(
    137         total=len(self.jobs),
    138         desc=self.desc,
    139         disable=not self.show_progress,
    140     ) as internal_pbar:
--> 141         await self._process_coroutines(
    142             self.jobs, internal_pbar, results, max_workers
    143         )
    144 else:
    145     await self._process_coroutines(
    146         self.jobs, self.pbar, results, max_workers
    147     )

File ~/Library/Caches/pypoetry/virtualenvs/siemens-rag-65vf45KI-py3.10/lib/python3.10/site-packages/ragas/executor.py:1
91, in Executor._process_coroutines(self, jobs, pbar, results, max_workers)
    189 coroutines = [afunc(*args, **kwargs) for afunc, args, kwargs, _ in jobs]
    190 for future in await as_completed(coroutines, max_workers):
--> 191     result = await future
    192     results.append(result)
    193     pbar.update(1)

File ~/.pyenv/versions/3.10.6/lib/python3.10/asyncio/tasks.py:571, in as_completed.<locals>._wait_for_one()
    568 if f is None:
    569     # Dummy value from _on_timeout().
    570     raise exceptions.TimeoutError
--> 571 return f.result()

File ~/.pyenv/versions/3.10.6/lib/python3.10/asyncio/futures.py:201, in Future.result(self)
    199 self.__log_traceback = False
    200 if self._exception is not None:
--> 201     raise self._exception.with_traceback(self._exception_tb)
    202 return self._result

File ~/.pyenv/versions/3.10.6/lib/python3.10/asyncio/tasks.py:232, in Task.__step(***failed resolving arguments***)
    228 try:
    229     if exc is None:
    230         # We use the `send` method directly, because coroutines
    231         # don't have `__iter__` and `__next__` methods.
--> 232         result = coro.send(None)
    233     else:
    234         result = coro.throw(exc)

File ~/Library/Caches/pypoetry/virtualenvs/siemens-rag-65vf45KI-py3.10/lib/python3.10/site-packages/ragas/executor.py:4
8, in as_completed.<locals>.sema_coro(coro)
     46 async def sema_coro(coro):
     47     async with semaphore:
---> 48         return await coro

File ~/Library/Caches/pypoetry/virtualenvs/siemens-rag-65vf45KI-py3.10/lib/python3.10/site-packages/ragas/executor.py:1
00, in Executor.wrap_callable_with_index.<locals>.wrapped_callable_async(*args, **kwargs)
     98 except Exception as e:
     99     if self.raise_exceptions:
--> 100         raise e
    101     else:
    102         exec_name = type(e).__name__

File ~/Library/Caches/pypoetry/virtualenvs/siemens-rag-65vf45KI-py3.10/lib/python3.10/site-packages/ragas/executor.py:9
6, in Executor.wrap_callable_with_index.<locals>.wrapped_callable_async(*args, **kwargs)
     92 async def wrapped_callable_async(
     93     *args, **kwargs
     94 ) -> t.Tuple[int, t.Callable | float]:
     95     try:
---> 96         result = await callable(*args, **kwargs)
     97         return counter, result
     98     except Exception as e:

File ~/Library/Caches/pypoetry/virtualenvs/siemens-rag-65vf45KI-py3.10/lib/python3.10/site-packages/ragas/testset/synth
esizers/base.py:94, in BaseSynthesizer.generate_scenarios(self, n, knowledge_graph, persona_list, callbacks)
     88 callbacks = callbacks or []
     89 scenario_generation_rm, scenario_generation_group = new_group(
     90     name=self.name,
     91     inputs={"n": n, "knowledge_graph": str(knowledge_graph)},
     92     callbacks=callbacks,
     93 )
---> 94 scenarios = await self._generate_scenarios(
     95     n, knowledge_graph, persona_list, scenario_generation_group
     96 )
     97 scenario_generation_rm.on_chain_end(outputs={"scenarios": scenarios})
     98 return scenarios

File ~/Library/Caches/pypoetry/virtualenvs/siemens-rag-65vf45KI-py3.10/lib/python3.10/site-packages/ragas/testset/synth
esizers/single_hop/specific.py:110, in SingleHopSpecificQuerySynthesizer._generate_scenarios(self, n, knowledge_graph, 
persona_list, callbacks)
    106     prompt_input = ThemesPersonasInput(themes=themes, personas=persona_list)
    107     persona_concepts = await self.theme_persona_matching_prompt.generate(
    108         data=prompt_input, llm=self.llm, callbacks=callbacks
    109     )
--> 110     base_scenarios = self.prepare_combinations(
    111         node,
    112         themes,
    113         personas=persona_list,
    114         persona_concepts=persona_concepts.mapping,
    115     )
    116     scenarios.extend(self.sample_combinations(base_scenarios, samples_per_node))
    118 return scenarios

File ~/Library/Caches/pypoetry/virtualenvs/siemens-rag-65vf45KI-py3.10/lib/python3.10/site-packages/ragas/testset/synth
esizers/single_hop/base.py:65, in SingleHopQuerySynthesizer.prepare_combinations(self, node, terms, personas, persona_c
oncepts)
     63 concepts = [concept.lower() for concept in concepts]
     64 if any(term.lower() in concepts for term in terms):
---> 65     if persona_list[persona]:
     66         valid_personas.append(persona_list[persona])
     67 sample["personas"] = valid_personas

File ~/Library/Caches/pypoetry/virtualenvs/siemens-rag-65vf45KI-py3.10/lib/python3.10/site-packages/ragas/testset/perso
na.py:59, in PersonaList.__getitem__(self, key)
     57     if persona.name == key:
     58         return persona
---> 59 raise KeyError(f"No persona found with name '{key}'")

KeyError: "No persona found with name 'Railway Safety Engineer 1'"
```

**Expected behavior**
A clear and concise description of what you expected to happen.


**Additional context**
Add any other context about the problem here.

<!-- PS: bugs suck but is also part of the process. We sincerely apologies for breaking your flow because of it, but don't worry, we got your back ❤️. We will get this fixed as fast as we can and thanks for helping us out by reporting it 🙏. -->


This also happens, when supplying a persona list:
```py
persona_list = generate_personas_from_kg(kg=kg, llm=ragas_llm, num_personas=5)

# Create generator
generator = TestsetGenerator(
	llm=ragas_llm,
	embedding_model=ragas_embedding,
	knowledge_graph=kg,
	persona_list=persona_list,
)
```

The Persona list look like this:
```sh
[Persona(name='Railway Safety Engineer', role_description='Responsible for identifying, addressing, and preventing comp
lex safety issues in the railway system, with a focus on enhancing safety culture and implementing international IT sec
urity standards.'),
 Persona(name='Railway Safety Engineer', role_description='Responsible for ensuring the safety and optimal functioning 
of rail traffic signalling systems, making critical decisions on reliability, availability, maintainability, and implem
enting counter measures to minimize risks.'),
 Persona(name='Railway Safety Engineer', role_description='Responsible for ensuring the safety of railway operations by
 implementing and analyzing various fail safety strategies, including inherent, composite, and reactive fail safety.'),
 Persona(name='Safety Systems Engineer', role_description='Responsible for analyzing hazards, specifying system archite
cture, determining safety requirements based on Safety Integrity Level (SIL) and hazard rates, ensuring compliance with
 these safety requirements, and addressing systematic errors in the design of safety signaling systems.'),
 Persona(name='Railway Safety Engineer', role_description='Responsible for implementing and maintaining safety signalli
ng systems in railway engineering, utilizing flexible standards and non-discriminatory risk analysis methods to ensure 
system safety and regulatory compliance.')]
```

I dont understand why the error message references a different persona name:
```sh
No persona found with name 'Railway Safety Engineer (1)'
```

The issue lies with the fact, that each Persona has the same name and therefore the mapping of themes and personas leads to errors as they are referenced with numbers by the llm. Solution is to use less personas or larger/more diverse documents