Repository: Giskard-AI/giskard-oss
Issue: how to fully change the LLM model and embedding model from default openAI to AzureAI?
Root cause: Documentation Deficiency

## Checklist

<!-- Mark with an `x` all the checkboxes that apply (like `[x]`) -->

- [x] I've searched the project's [`issues`](https://github.com/Giskard-AI/giskard/issues?q=is%3Aissue).

## ❓ Question

<!-- What is your question -->
First of all, thanks to giskard team sharing the convenient tool.
As the document showed => https://docs.giskard.ai/en/latest/open_source/setting_up/index.html 
I just want to confirm that I try to set below settings in three giskard smaple applications in order to replace the LLM model and embedding model from default opanAI to AzureAI ( **[LLM scan] ,[RAG testset generation],  [evaluate your RAG]** ) 
```
os.environ["AZURE_API_KEY"] = "8******************************a"
os.environ["AZURE_API_BASE"] = "https://*******.openai.azure.com"
os.environ["AZURE_API_VERSION"] = "2024-05-01-preview"

giskard.llm.set_llm_model("azure/gpt-4o")
giskard.llm.set_embedding_model("azure/text-embedding-3-small")
```


I found out only **[RAG testset generation]** application can truly use my Azure Open AI service.
# [RAG testset generation] full code, this code can successfully generate QA file by using my Azure_API_KEY.
```
import pandas as pd
import giskard
from openai import OpenAI
from giskard.llm.client.openai import OpenAIClient
from giskard.rag import KnowledgeBase, generate_testset, QATestset
from llama_index.readers.file import PyMuPDFReader
from llama_index.core.node_parser import SentenceSplitter
import os
os.environ["AZURE_API_KEY"] = "8******************************a"
os.environ["AZURE_API_BASE"] = "https://*******.openai.azure.com"
os.environ["AZURE_API_VERSION"] = "2024-05-01-preview"
giskard.llm.set_llm_model("azure/gpt-4o")
giskard.llm.set_embedding_model("azure/text-embedding-3-small")

loader = PyMuPDFReader()
splitter = SentenceSplitter(chunk_size=512)

ipcc_documents = loader.load(file_path="./ipcc_report.pdf")
text_nodes = splitter(ipcc_documents)

knowledge_base_df = pd.DataFrame([node.text for node in text_nodes], columns=["text"])
knowledge_base_df.head()
knowledge_base = KnowledgeBase(knowledge_base_df)
print(knowledge_base)

 testset = generate_testset(
     knowledge_base,
     num_questions=30,
     language="en",
     agent_description="A chatbot answering questions about the IPCC report",
 )
testset.save("test_IPCC.jsonl")
```

# Howerver the **[LLM scan]**  application showed **"Did not find openai_api_key"** error

```
  File "C:\mypython3\giskardBuildRAG\Lib\site-packages\langchain_core\load\serializable.py", line 125, in __init__
    super().__init__(*args, **kwargs)
  File "C:\mypython3\giskardBuildRAG\Lib\site-packages\pydantic\main.py", line 214, in __init__
    validated_self = self.__pydantic_validator__.validate_python(data, self_instance=self)
                     ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
pydantic_core._pydantic_core.ValidationError: 1 validation error for OpenAI
  Value error, Did not find openai_api_key, please add an environment variable `OPENAI_API_KEY` which contains it, or pass `openai_api_key` as a named parameter. [type=value_error, input_value={'temperature': 0, 'model...ne, 'http_client': None}, input_type=dict]
    For further information visit https://errors.pydantic.dev/2.10/v/value_error
```
# [LLM scan] full code, can only successed when using openAI key .
```
import os
import giskard
from pathlib import Path

import openai
import pandas as pd
from langchain.llms import OpenAI
from langchain.chains.base import Chain
from langchain.vectorstores import FAISS
from langchain.prompts import PromptTemplate
# from langchain.embeddings import OpenAIEmbeddings
from langchain.embeddings import AzureOpenAIEmbeddings
from langchain.document_loaders import PyPDFLoader
from langchain.chains import RetrievalQA, load_chain
from langchain.text_splitter import RecursiveCharacterTextSplitter
from giskard import Dataset, Model, scan

os.environ["AZURE_API_KEY"] = "8******************************a"
os.environ["AZURE_API_BASE"] = "https://*******.openai.azure.com"
os.environ["AZURE_API_VERSION"] = "2024-05-01-preview"
giskard.llm.set_llm_model("azure/gpt-4o")
giskard.llm.set_embedding_model("azure/text-embedding-3-small")

pd.set_option("display.max_colwidth", None)

from langchain.chat_models import ChatOpenAI



IPCC_REPORT_URL = "https://www.ipcc.ch/report/ar6/syr/downloads/report/IPCC_AR6_SYR_LongerReport.pdf"

LLM_NAME = "gemini/gemini-1.5-pro"

TEXT_COLUMN_NAME = "query"

PROMPT_TEMPLATE = """You are the Climate Assistant, a helpful AI assistant made by Giskard.
Your task is to answer common questions on climate change.
You will be given a question and relevant excerpts from the IPCC Climate Change Synthesis Report (2023).
Please provide short and clear answers based on the provided context. Be polite and helpful.

Context:
{context}

Question:
{question}

Your answer:
"""

def get_context_storage() -> FAISS:
    """Initialize a vector storage of embedded IPCC report chunks (context)."""
    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100, add_start_index=True)
    docs = PyPDFLoader(IPCC_REPORT_URL).load_and_split(text_splitter)
    db = FAISS.from_documents(docs, OpenAIEmbeddings())
    return db


# Create the chain.
llm = OpenAI(temperature=0)
prompt = PromptTemplate(template=PROMPT_TEMPLATE, input_variables=["question", "context"])
climate_qa_chain = RetrievalQA.from_llm(llm=llm, retriever=get_context_storage().as_retriever(), prompt=prompt)

# Test the chain.
climate_qa_chain("Is sea level rise avoidable? When will it stop?")

# Define a custom Giskard model wrapper for the serialization.
class FAISSRAGModel(Model):
    def model_predict(self, df: pd.DataFrame) -> pd.DataFrame:
        return df[TEXT_COLUMN_NAME].apply(lambda x: self.model.run({"query": x}))

    def save_model(self, path: str):
        out_dest = Path(path)
        # Save the chain object
        self.model.save(out_dest.joinpath("model.json"))

        # Save the FAISS-based retriever
        db = self.model.retriever.vectorstore
        db.save_local(out_dest.joinpath("faiss"))

    @classmethod
    def load_model(cls, path: str) -> Chain:
        src = Path(path)

        # Load the FAISS-based retriever
        db = FAISS.load_local(src.joinpath("faiss"), OpenAIEmbeddings())

        # Load the chain, passing the retriever
        chain = load_chain(src.joinpath("model.json"), retriever=db.as_retriever())
        return chain


# Wrap the QA chain
giskard_model = FAISSRAGModel(
    model=climate_qa_chain,  # A prediction function that encapsulates all the data pre-processing steps and that could be executed with the dataset used by the scan.
    model_type="text_generation",  # Either regression, classification or text_generation.
    name="Climate Change Question Answering",  # Optional.
    description="This model answers any question about climate change based on IPCC reports",  # Is used to generate prompts during the scan.
    feature_names=[TEXT_COLUMN_NAME]  # Default: all columns of your dataset.
)

# Optional: Wrap a dataframe of sample input prompts to validate the model wrapping and to narrow specific tests' queries.
giskard_dataset = Dataset(pd.DataFrame({
    TEXT_COLUMN_NAME: [
        "According to the IPCC report, what are key risks in the Europe?",
        "Is sea level rise avoidable? When will it stop?"
    ]
}))

# Validate the wrapped model and dataset.
print(giskard_model.predict(giskard_dataset).prediction)

results = scan(giskard_model, giskard_dataset, only="hallucination")

# display(results)

test_suite = results.generate_test_suite("Test suite generated by scan")
test_suite.run()

with open("llm_scan_report.html", "w", encoding="utf-8") as f:
    f.write(results.to_html())
```

# **[evaluate your RAG]** also shows "**Could not load OpenAI embedding model**" error message.
```
  File "C:\mypython3\giskardRagasEvaluate1\Lib\site-packages\llama_index\core\indices\vector_store\base.py", line 72, in __init__
    else Settings.embed_model
         ^^^^^^^^^^^^^^^^^^^^
  File "C:\mypython3\giskardRagasEvaluate1\Lib\site-packages\llama_index\core\settings.py", line 64, in embed_model
    self._embed_model = resolve_embed_model("default")
                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\mypython3\giskardRagasEvaluate1\Lib\site-packages\llama_index\core\embeddings\utils.py", line 66, in resolve_embed_model
    raise ValueError(
ValueError:
******
Could not load OpenAI embedding model. If you intended to use OpenAI, please check your OPENAI_API_KEY.
Original error:
No API key found for OpenAI.
Please set either the OPENAI_API_KEY environment variable or openai.api_key prior to initialization.
API keys can be found or created at https://platform.openai.com/account/api-keys

Consider using embed_model='local'.
Visit our documentation for more embedding options: https://docs.llamaindex.ai/en/stable/module_guides/models/embeddings.html#modules
******
```
# **[evaluate your RAG]** full code, can only successed when using openAI key .
```
import pandas as pd
import warnings
import giskard
from openai import OpenAI
from giskard.llm.client.openai import OpenAIClient
import os
from llama_index.core import VectorStoreIndex
from llama_index.core.node_parser import SentenceSplitter
from llama_index.readers.file import PyMuPDFReader
from llama_index.core.base.llms.types import ChatMessage, MessageRole
from giskard.rag import KnowledgeBase, generate_testset, QATestset
from giskard.rag import evaluate, RAGReport, AgentAnswer
from giskard.rag.metrics.ragas_metrics import ragas_context_recall, ragas_context_precision, ragas_faithfulness, ragas_answer_relevancy
from giskard.llm import set_llm_model, set_embedding_model
from giskard.llm import set_llm_model, set_embedding_model
pd.set_option("display.max_colwidth", 400)
warnings.filterwarnings('ignore')

os.environ["AZURE_API_KEY"] = "8******************************a"
os.environ["AZURE_API_BASE"] = "https://*******.openai.azure.com"
os.environ["AZURE_API_VERSION"] = "2024-05-01-preview"
giskard.llm.set_llm_model("azure/gpt-4o")
giskard.llm.set_embedding_model("azure/text-embedding-3-small")

loader = PyMuPDFReader()
ipcc_documents = loader.load(file_path="./2023esgDoc.pdf")

splitter = SentenceSplitter(chunk_size=512)
index = VectorStoreIndex.from_documents(ipcc_documents, transformations=[splitter])
chat_engine = index.as_chat_engine()

text_nodes = splitter(ipcc_documents)
knowledge_base_df = pd.DataFrame([node.text for node in text_nodes], columns=["text"])
knowledge_base = KnowledgeBase(knowledge_base_df)
testset = QATestset.load("test_ESG_employee_welfare_3.jsonl")

def answer_fn(question, history=None):
    if history:
        answer = chat_engine.chat(
            question,
            chat_history=[
                ChatMessage(
                    role=MessageRole.USER if msg["role"] == "user" else MessageRole.ASSISTANT,
                    content=msg["content"]
                ) for msg in history
            ]
        )
    else:
        answer = chat_engine.chat(question, chat_history=[])

    return AgentAnswer(
        message=answer.response,
        documents=[source.content for source in answer.sources]
    )

report = evaluate(answer_fn,
                testset=testset,
                knowledge_base=knowledge_base,
                metrics=[ragas_context_recall, ragas_context_precision,ragas_faithfulness, ragas_answer_relevancy],)



report.save("esg_reportout.html")
```
 ## Can anyone give me some advice? Thanks.



Hello @jerry0688 

On the code snippet that you shared you're setting up the giskard evaluator to use Azure OpenAI properly. 

However, you're trying to evaluate an LLM agent that use OpenAI so it fails because the OpenAI API key is not setup properly. 

Basically when using giskard there is two things to setup:
- The evaluator LLM provider (in your case Azure OpenAI)
- The agent to evaluate (in this case example IPCC climate agent built on OpenAI)


Here is a code snippet that update the IPCC climate agent to use Azure OpenAI:
```python
import os
from pathlib import Path

import pandas as pd
from langchain.chains import RetrievalQA, load_chain
from langchain.chains.base import Chain
from langchain.document_loaders import PyPDFLoader
from langchain.prompts import PromptTemplate
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain.vectorstores import FAISS
from langchain_openai import AzureOpenAI
from langchain_openai import AzureOpenAIEmbeddings

import giskard
from giskard import Dataset, Model, scan

os.environ["AZURE_API_KEY"] = "8******************************a"
os.environ["AZURE_API_BASE"] = "https://*******.openai.azure.com"
os.environ["AZURE_API_VERSION"] = "2024-05-01-preview"
giskard.llm.set_llm_model("azure/gpt-4o")
giskard.llm.set_embedding_model("azure/text-embedding-3-small")

pd.set_option("display.max_colwidth", None)

IPCC_REPORT_URL = "https://www.ipcc.ch/report/ar6/syr/downloads/report/IPCC_AR6_SYR_LongerReport.pdf"

LLM_NAME = "gemini/gemini-1.5-pro"

TEXT_COLUMN_NAME = "query"

PROMPT_TEMPLATE = """You are the Climate Assistant, a helpful AI assistant made by Giskard.
Your task is to answer common questions on climate change.
You will be given a question and relevant excerpts from the IPCC Climate Change Synthesis Report (2023).
Please provide short and clear answers based on the provided context. Be polite and helpful.

Context:
{context}

Question:
{question}

Your answer:
"""

def get_context_storage() -> FAISS:
    """Initialize a vector storage of embedded IPCC report chunks (context)."""
    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=100, add_start_index=True)
    docs = PyPDFLoader(IPCC_REPORT_URL).load_and_split(text_splitter)
    db = FAISS.from_documents(docs, AzureOpenAIEmbeddings(model="text-embedding-3-large")) # TODO: adjust model to desired one
    return db


# Create the chain.
llm = AzureOpenAI(deployment_name="gpt-35-turbo-instruct-0914", temperature=0) # TODO: adjust deployment_name to desired one
prompt = PromptTemplate(template=PROMPT_TEMPLATE, input_variables=["question", "context"])
climate_qa_chain = RetrievalQA.from_llm(llm=llm, retriever=get_context_storage().as_retriever(), prompt=prompt)

# Test the chain.
climate_qa_chain("Is sea level rise avoidable? When will it stop?")

# Define a custom Giskard model wrapper for the serialization.
class FAISSRAGModel(Model):
    def model_predict(self, df: pd.DataFrame) -> pd.DataFrame:
        return df[TEXT_COLUMN_NAME].apply(lambda x: self.model.run({"query": x}))

    def save_model(self, path: str):
        out_dest = Path(path)
        # Save the chain object
        self.model.save(out_dest.joinpath("model.json"))

        # Save the FAISS-based retriever
        db = self.model.retriever.vectorstore
        db.save_local(out_dest.joinpath("faiss"))

    @classmethod
    def load_model(cls, path: str) -> Chain:
        src = Path(path)

        # Load the FAISS-based retriever
        db = FAISS.load_local(src.joinpath("faiss"), OpenAIEmbeddings())

        # Load the chain, passing the retriever
        chain = load_chain(src.joinpath("model.json"), retriever=db.as_retriever())
        return chain


# Wrap the QA chain
giskard_model = FAISSRAGModel(
    model=climate_qa_chain,  # A prediction function that encapsulates all the data pre-processing steps and that could be executed with the dataset used by the scan.
    model_type="text_generation",  # Either regression, classification or text_generation.
    name="Climate Change Question Answering",  # Optional.
    description="This model answers any question about climate change based on IPCC reports",  # Is used to generate prompts during the scan.
    feature_names=[TEXT_COLUMN_NAME]  # Default: all columns of your dataset.
)

# Optional: Wrap a dataframe of sample input prompts to validate the model wrapping and to narrow specific tests' queries.
giskard_dataset = Dataset(pd.DataFrame({
    TEXT_COLUMN_NAME: [
        "According to the IPCC report, what are key risks in the Europe?",
        "Is sea level rise avoidable? When will it stop?"
    ]
}))

# Validate the wrapped model and dataset.
print(giskard_model.predict(giskard_dataset).prediction)

results = scan(giskard_model, giskard_dataset, only="hallucination")

# display(results)

test_suite = results.generate_test_suite("Test suite generated by scan")
test_suite.run()

with open("llm_scan_report.html", "w", encoding="utf-8") as f:
    f.write(results.to_html())
```