Repository: explodinggradients/ragas
Issue: Using AzureOpenAI raises InvalidRequestError
Root cause: Configuration Error

**Describe the bug**
Getting **InvalidRequestError: Must provide an 'engine' or 'deployment_id' parameter to create a <class 'openai.api_resources.chat_completion.ChatCompletion'>** when trying to reproduce Ragas code from https://blog.langchain.dev/evaluating-rag-pipelines-with-ragas-langsmith/ using AzureOpenAI instead


Ragas version: 0.0.19
Python version: 3.8.18 

**Code to Reproduce**
from langchain.document_loaders import WebBaseLoader
from langchain.vectorstores import Qdrant
from langchain.embeddings import HuggingFaceEmbeddings
from langchain.text_splitter import CharacterTextSplitter, RecursiveCharacterTextSplitter
from langchain.chains import RetrievalQA
from langchain.llms import AzureOpenAI

import os

os.environ["OPENAI_API_TYPE"] = "azure"
os.environ["OPENAI_API_BASE"] = "<<>>"
os.environ["OPENAI_API_KEY"] = "<<>>"
os.environ["OPENAI_API_VERSION"] = "<<>>"

loader = WebBaseLoader("https://en.wikipedia.org/wiki/New_York_City")
loader.requests_kwargs = {'verify':False}
docs = loader.load()
text_splitter = RecursiveCharacterTextSplitter(
chunk_size=1000, chunk_overlap=0, separators=[" ", ",", "\n"]
)
docs = text_splitter.split_documents(docs)

embeddings = HuggingFaceEmbeddings(
    model_name="sentence-transformers/all-mpnet-base-v2"
)
doc_store = Qdrant.from_documents(
    docs, embeddings, path="/tmp/test", 
)

llm = AzureOpenAI(
*comment*   deployment_name="Davinci",
*comment*     engine = "Davinci",
*comment*      deployment_id = "text-davinci-003",
*comment*     model_name="text-davinci-003",
    model_kwargs={"engine":"Davinci", "deployment_id" : "text-davinci-003" },
)

qa_chain = RetrievalQA.from_chain_type(
    llm, retriever=doc_store.as_retriever(), return_source_documents=True
)

question = "How did New York City get its name?"
result = qa_chain({"query": question})

from ragas.metrics import faithfulness, answer_relevancy, context_relevancy, context_recall
from ragas.langchain import RagasEvaluatorChain

eval_chains = {
    m.name: RagasEvaluatorChain(metric=m) 
    for m in [faithfulness, answer_relevancy, context_relevancy, context_recall]
}

for name, eval_chain in eval_chains.items():
    score_name = f"{name}_score"
    print(f"{score_name}: {eval_chain(result)[score_name]}")

**Error trace**
---------------------------------------------------------------------------
InvalidRequestError                       Traceback (most recent call last)
Cell In[25], line 4
      2 for name, eval_chain in eval_chains.items():
      3     score_name = f"{name}_score"
----> 4     print(f"{score_name}: {eval_chain(result)[score_name]}")

File ~/opt/anaconda3/envs/llm/lib/python3.8/site-packages/langchain/chains/base.py:310, in Chain.__call__(self, inputs, return_only_outputs, callbacks, tags, metadata, run_name, include_run_info)
    308 except BaseException as e:
    309     run_manager.on_chain_error(e)
--> 310     raise e
    311 run_manager.on_chain_end(outputs)
    312 final_outputs: Dict[str, Any] = self.prep_outputs(
    313     inputs, outputs, return_only_outputs
    314 )

File ~/opt/anaconda3/envs/llm/lib/python3.8/site-packages/langchain/chains/base.py:304, in Chain.__call__(self, inputs, return_only_outputs, callbacks, tags, metadata, run_name, include_run_info)
    297 run_manager = callback_manager.on_chain_start(
    298     dumpd(self),
    299     inputs,
    300     name=run_name,
    301 )
    302 try:
    303     outputs = (
--> 304         self._call(inputs, run_manager=run_manager)
    305         if new_arg_supported
    306         else self._call(inputs)
    307     )
    308 except BaseException as e:
    309     run_manager.on_chain_error(e)

File ~/opt/anaconda3/envs/llm/lib/python3.8/site-packages/ragas/langchain/evalchain.py:70, in RagasEvaluatorChain._call(self, inputs, run_manager)
     68 question = inputs["query"]
     69 answer = inputs["result"]
---> 70 score = self.metric.score_single(
     71     {
     72         "question": question,
     73         "answer": answer,
     74         "contexts": contexts,
     75         "ground_truths": ground_truths,
     76     },
     77     callbacks=callbacks,
     78 )
     79 return {f"{self.metric.name}_score": score}

File ~/opt/anaconda3/envs/llm/lib/python3.8/site-packages/ragas/metrics/base.py:101, in Metric.score_single(self, ds_row, callbacks)
     98 # TODO: validation check if they are string
    100 ds = Dataset.from_dict({k: [v] for k, v in ds_row.items()})
--> 101 score = self._score_batch(
    102     ds, callback_group_name=self.name, callbacks=callbacks
    103 )
    105 return score[0]

File ~/opt/anaconda3/envs/llm/lib/python3.8/site-packages/ragas/metrics/faithfulness.py:87, in Faithfulness._score_batch(self, ds, callbacks, callback_group_name)
     84     human_prompt = LONG_FORM_ANSWER_PROMPT.format(question=q, answer=a)
     85     prompts.append(ChatPromptTemplate.from_messages([human_prompt]))
---> 87 result = self.llm.generate(prompts, callbacks=batch_group)
     88 list_statements: list[list[str]] = []
     89 for output in result.generations:
     90     # use only the first generation for each prompt

File ~/opt/anaconda3/envs/llm/lib/python3.8/site-packages/ragas/llms/base.py:127, in LangchainLLM.generate(self, prompts, n, temperature, callbacks)
    124     self.llm.temperature = temperature
    126 if self.llm_supports_completions(self.llm):
--> 127     return self.generate_multiple_completions(prompts, n, callbacks)
    128 else:  # call generate_completions n times to mimic multiple completions
    129     list_llmresults = run_async_tasks(
    130         [self.generate_completions(prompts, callbacks) for _ in range(n)]
    131     )

File ~/opt/anaconda3/envs/llm/lib/python3.8/site-packages/ragas/llms/base.py:93, in LangchainLLM.generate_multiple_completions(self, prompts, n, callbacks)
     91 else:  # if BaseChatModel
     92     ps = [p.format_messages() for p in prompts]
---> 93     result = self.llm.generate(ps, callbacks=callbacks)
     94 self.llm.n = old_n
     96 return result

File ~/opt/anaconda3/envs/llm/lib/python3.8/site-packages/langchain/chat_models/base.py:349, in BaseChatModel.generate(self, messages, stop, callbacks, tags, metadata, run_name, **kwargs)
    347         if run_managers:
    348             run_managers[i].on_llm_error(e)
--> 349         raise e
    350 flattened_outputs = [
    351     LLMResult(generations=[res.generations], llm_output=res.llm_output)
    352     for res in results
    353 ]
    354 llm_output = self._combine_llm_outputs([res.llm_output for res in results])

File ~/opt/anaconda3/envs/llm/lib/python3.8/site-packages/langchain/chat_models/base.py:339, in BaseChatModel.generate(self, messages, stop, callbacks, tags, metadata, run_name, **kwargs)
    336 for i, m in enumerate(messages):
    337     try:
    338         results.append(
--> 339             self._generate_with_cache(
    340                 m,
    341                 stop=stop,
    342                 run_manager=run_managers[i] if run_managers else None,
    343                 **kwargs,
    344             )
    345         )
    346     except BaseException as e:
    347         if run_managers:

File ~/opt/anaconda3/envs/llm/lib/python3.8/site-packages/langchain/chat_models/base.py:492, in BaseChatModel._generate_with_cache(self, messages, stop, run_manager, **kwargs)
    488     raise ValueError(
    489         "Asked to cache, but no cache found at `langchain.cache`."
    490     )
    491 if new_arg_supported:
--> 492     return self._generate(
    493         messages, stop=stop, run_manager=run_manager, **kwargs
    494     )
    495 else:
    496     return self._generate(messages, stop=stop, **kwargs)

File ~/opt/anaconda3/envs/llm/lib/python3.8/site-packages/langchain/chat_models/openai.py:360, in ChatOpenAI._generate(self, messages, stop, run_manager, stream, **kwargs)
    358 message_dicts, params = self._create_message_dicts(messages, stop)
    359 params = {**params, **kwargs}
--> 360 response = self.completion_with_retry(
    361     messages=message_dicts, run_manager=run_manager, **params
    362 )
    363 return self._create_chat_result(response)

File ~/opt/anaconda3/envs/llm/lib/python3.8/site-packages/langchain/chat_models/openai.py:299, in ChatOpenAI.completion_with_retry(self, run_manager, **kwargs)
    295 @retry_decorator
    296 def _completion_with_retry(**kwargs: Any) -> Any:
    297     return self.client.create(**kwargs)
--> 299 return _completion_with_retry(**kwargs)

File ~/opt/anaconda3/envs/llm/lib/python3.8/site-packages/tenacity/__init__.py:289, in BaseRetrying.wraps.<locals>.wrapped_f(*args, **kw)
    287 @functools.wraps(f)
    288 def wrapped_f(*args: t.Any, **kw: t.Any) -> t.Any:
--> 289     return self(f, *args, **kw)

File ~/opt/anaconda3/envs/llm/lib/python3.8/site-packages/tenacity/__init__.py:379, in Retrying.__call__(self, fn, *args, **kwargs)
    377 retry_state = RetryCallState(retry_object=self, fn=fn, args=args, kwargs=kwargs)
    378 while True:
--> 379     do = self.iter(retry_state=retry_state)
    380     if isinstance(do, DoAttempt):
    381         try:

File ~/opt/anaconda3/envs/llm/lib/python3.8/site-packages/tenacity/__init__.py:314, in BaseRetrying.iter(self, retry_state)
    312 is_explicit_retry = fut.failed and isinstance(fut.exception(), TryAgain)
    313 if not (is_explicit_retry or self.retry(retry_state)):
--> 314     return fut.result()
    316 if self.after is not None:
    317     self.after(retry_state)

File ~/opt/anaconda3/envs/llm/lib/python3.8/concurrent/futures/_base.py:437, in Future.result(self, timeout)
    435     raise CancelledError()
    436 elif self._state == FINISHED:
--> 437     return self.__get_result()
    439 self._condition.wait(timeout)
    441 if self._state in [CANCELLED, CANCELLED_AND_NOTIFIED]:

File ~/opt/anaconda3/envs/llm/lib/python3.8/concurrent/futures/_base.py:389, in Future.__get_result(self)
    387 if self._exception:
    388     try:
--> 389         raise self._exception
    390     finally:
    391         # Break a reference cycle with the exception in self._exception
    392         self = None

File ~/opt/anaconda3/envs/llm/lib/python3.8/site-packages/tenacity/__init__.py:382, in Retrying.__call__(self, fn, *args, **kwargs)
    380 if isinstance(do, DoAttempt):
    381     try:
--> 382         result = fn(*args, **kwargs)
    383     except BaseException:  # noqa: B902
    384         retry_state.set_exception(sys.exc_info())  # type: ignore[arg-type]

File ~/opt/anaconda3/envs/llm/lib/python3.8/site-packages/langchain/chat_models/openai.py:297, in ChatOpenAI.completion_with_retry.<locals>._completion_with_retry(**kwargs)
    295 @retry_decorator
    296 def _completion_with_retry(**kwargs: Any) -> Any:
--> 297     return self.client.create(**kwargs)

File ~/opt/anaconda3/envs/llm/lib/python3.8/site-packages/openai/api_resources/chat_completion.py:25, in ChatCompletion.create(cls, *args, **kwargs)
     23 while True:
     24     try:
---> 25         return super().create(*args, **kwargs)
     26     except TryAgain as e:
     27         if timeout is not None and time.time() > start + timeout:

File ~/opt/anaconda3/envs/llm/lib/python3.8/site-packages/openai/api_resources/abstract/engine_api_resource.py:151, in EngineAPIResource.create(cls, api_key, api_base, api_type, request_id, api_version, organization, **params)
    129 @classmethod
    130 def create(
    131     cls,
   (...)
    138     **params,
    139 ):
    140     (
    141         deployment_id,
    142         engine,
    143         timeout,
    144         stream,
    145         headers,
    146         request_timeout,
    147         typed_api_type,
    148         requestor,
    149         url,
    150         params,
--> 151     ) = cls.__prepare_create_request(
    152         api_key, api_base, api_type, api_version, organization, **params
    153     )
    155     response, _, api_key = requestor.request(
    156         "post",
    157         url,
   (...)
    162         request_timeout=request_timeout,
    163     )
    165     if stream:
    166         # must be an iterator

File ~/opt/anaconda3/envs/llm/lib/python3.8/site-packages/openai/api_resources/abstract/engine_api_resource.py:85, in EngineAPIResource.__prepare_create_request(cls, api_key, api_base, api_type, api_version, organization, **params)
     83 if typed_api_type in (util.ApiType.AZURE, util.ApiType.AZURE_AD):
     84     if deployment_id is None and engine is None:
---> 85         raise error.InvalidRequestError(
     86             "Must provide an 'engine' or 'deployment_id' parameter to create a %s"
     87             % cls,
     88             "engine",
     89         )
     90 else:
     91     if model is None and engine is None:

InvalidRequestError: Must provide an 'engine' or 'deployment_id' parameter to create a <class 'openai.api_resources.chat_completion.ChatCompletion'>

**Expected behavior**
 output
faithfulness_score: 1.0
answer_relevancy_score: 0.9193459596511587
context_relevancy_score: 0.07480974380786602
context_recall_score: 0.9193459596511587


**Additional context**
Add any other context about the problem here.


Issue was resolved following guide here https://docs.ragas.io/en/latest/howtos/customisations/azure-openai.html