Evaluation

We have modules for both LLM-based evaluation and retrieval-based
evaluation.
Evaluation modules.
class llama_index.evaluation.BaseEvaluator
   Base Evaluator class.
   abstract async aevaluate(query: Optional[str] = None, response: Optional[str] = None, contexts: Optional[Sequence[str]] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string, retrieved contexts, and
      generated response string.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
   async aevaluate_response(query: Optional[str] = None, response: Optional[Response] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string and generated Response object.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
   evaluate(query: Optional[str] = None, response: Optional[str] = None, contexts: Optional[Sequence[str]] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string, retrieved contexts, and
      generated response string.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
   evaluate_response(query: Optional[str] = None, response: Optional[Response] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string and generated Response object.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
pydantic model llama_index.evaluation.BaseRetrievalEvaluator
   Base Retrieval Evaluator class.
      {
         "title": "BaseRetrievalEvaluator",
         "description": "Base Retrieval Evaluator class.",
         "type": "object",
         "properties": {
            "metrics": {
               "title": "Metrics"
            }
         }
      }
   Config:
      * **arbitrary_types_allowed**: *bool = True*
   Fields:
      * "metrics (List[llama_index.evaluation.retrieval.metrics_base.
        BaseRetrievalMetric])"
   field metrics: List[BaseRetrievalMetric] [Required]
      List of metrics to evaluate
   async aevaluate(query: str, expected_ids: List[str], **kwargs: Any) -> RetrievalEvalResult
      Run evaluation with query string, retrieved contexts, and
      generated response string.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
   async aevaluate_dataset(dataset: EmbeddingQAFinetuneDataset, workers: int = 2, show_progress: bool = False, **kwargs: Any) -> List[RetrievalEvalResult]
      Run evaluation with dataset.
   classmethod construct(_fields_set: Optional[SetStr] = None, **values: Any) -> Model
      Creates a new model setting __dict__ and __fields_set__ from
      trusted or pre-validated data. Default values are respected, but
      no other validation is performed. Behaves as if *Config.extra =
      'allow'* was set since it adds all passed values
   copy(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, update: Optional[DictStrAny] = None, deep: bool = False) -> Model
      Duplicate a model, optionally choose which fields to include,
      exclude and change.
      Parameters:
         * **include** -- fields to include in new model
         * **exclude** -- fields to exclude from new model, as with
           values this takes precedence over include
         * **update** -- values to change/add in the new model. Note:
           the data is not validated before creating the new model:
           you should trust this data
         * **deep** -- set to *True* to make a deep copy of the model

Evaluation

      Returns:
         new model instance
   dict(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, by_alias: bool = False, skip_defaults: Optional[bool] = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False) -> DictStrAny
      Generate a dictionary representation of the model, optionally
      specifying which fields to include or exclude.
   evaluate(query: str, expected_ids: List[str], **kwargs: Any) -> RetrievalEvalResult
      Run evaluation results with query string and expected ids.
      Parameters:
         * **query** (*str*) -- Query string
         * **expected_ids** (*List**[**str**]*) -- Expected ids
      Returns:
         Evaluation result
      Return type:
         RetrievalEvalResult
   classmethod from_metric_names(metric_names: List[str], **kwargs: Any) -> BaseRetrievalEvaluator
      Create evaluator from metric names.
      Parameters:
         * **metric_names** (*List**[**str**]*) -- List of metric
           names
         * ****kwargs** -- Additional arguments for the evaluator
   classmethod from_orm(obj: Any) -> Model
   json(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, by_alias: bool = False, skip_defaults: Optional[bool] = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False, encoder: Optional[Callable[[Any], Any]] = None, models_as_dict: bool = True, **dumps_kwargs: Any) -> unicode
      Generate a JSON representation of the model, *include* and
      *exclude* arguments as per *dict()*.
      *encoder* is an optional function to supply as *default* to
      json.dumps(), other arguments as per *json.dumps()*.
   classmethod parse_file(path: Union[str, Path], *, content_type: unicode = None, encoding: unicode = 'utf8', proto: Protocol = None, allow_pickle: bool = False) -> Model
   classmethod parse_obj(obj: Any) -> Model
   classmethod parse_raw(b: Union[str, bytes], *, content_type: unicode = None, encoding: unicode = 'utf8', proto: Protocol = None, allow_pickle: bool = False) -> Model
   classmethod schema(by_alias: bool = True, ref_template: unicode = '#/definitions/{model}') -> DictStrAny
   classmethod schema_json(*, by_alias: bool = True, ref_template: unicode = '#/definitions/{model}', **dumps_kwargs: Any) -> unicode
   classmethod update_forward_refs(**localns: Any) -> None
      Try to update ForwardRefs on fields based on this Model,
      globalns and localns.
   classmethod validate(value: Any) -> Model
class llama_index.evaluation.BatchEvalRunner(evaluators: Dict[str, BaseEvaluator], workers: int = 2, show_progress: bool = False)
   Batch evaluation runner.
   Parameters:
      * **evaluators** (*Dict**[**str**, **BaseEvaluator**]*) --
        Dictionary of evaluators.
      * **workers** (*int*) -- Number of workers to use for
        parallelization. Defaults to 2.
      * **show_progress** (*bool*) -- Whether to show progress bars.
        Defaults to False.
   async aevaluate_queries(query_engine: BaseQueryEngine, queries: Optional[List[str]] = None, **eval_kwargs_lists: Dict[str, Any]) -> Dict[str, List[EvaluationResult]]

Evaluation

      Evaluate queries.
      Parameters:
         * **query_engine** (*BaseQueryEngine*) -- Query engine.
         * **queries** (*Optional**[**List**[**str**]**]*) -- List of
           query strings. Defaults to None.
         * ****eval_kwargs_lists** (*Dict**[**str**, **Any**]*) --
           Dict of lists of kwargs to pass to evaluator. Defaults to
           None.
   async aevaluate_response_strs(queries: Optional[List[str]] = None, response_strs: Optional[List[str]] = None, contexts_list: Optional[List[List[str]]] = None, **eval_kwargs_lists: List) -> Dict[str, List[EvaluationResult]]
      Evaluate query, response pairs.
      This evaluates queries, responses, contexts as string inputs.
      Can supply additional kwargs to the evaluator in
      eval_kwargs_lists.
      Parameters:
         * **queries** (*Optional**[**List**[**str**]**]*) -- List of
           query strings. Defaults to None.
         * **response_strs** (*Optional**[**List**[**str**]**]*) --
           List of response strings. Defaults to None.
         * **contexts_list**
           (*Optional**[**List**[**List**[**str**]**]**]*) -- List of
           context lists. Defaults to None.
         * ****eval_kwargs_lists** (*Dict**[**str**, **Any**]*) --
           Dict of lists of kwargs to pass to evaluator. Defaults to
           None.
   async aevaluate_responses(queries: Optional[List[str]] = None, responses: Optional[List[Response]] = None, **eval_kwargs_lists: Dict[str, Any]) -> Dict[str, List[EvaluationResult]]
      Evaluate query, response pairs.
      This evaluates queries and response objects.
      Parameters:
         * **queries** (*Optional**[**List**[**str**]**]*) -- List of
           query strings. Defaults to None.
         * **responses** (*Optional**[**List**[**Response**]**]*) --
           List of response objects. Defaults to None.
         * ****eval_kwargs_lists** (*Dict**[**str**, **Any**]*) --
           Dict of lists of kwargs to pass to evaluator. Defaults to
           None.
   evaluate_queries(query_engine: BaseQueryEngine, queries: Optional[List[str]] = None, **eval_kwargs_lists: Dict[str, Any]) -> Dict[str, List[EvaluationResult]]
      Evaluate queries.
      Sync version of aevaluate_queries.
   evaluate_response_strs(queries: Optional[List[str]] = None, response_strs: Optional[List[str]] = None, contexts_list: Optional[List[List[str]]] = None, **eval_kwargs_lists: List) -> Dict[str, List[EvaluationResult]]
      Evaluate query, response pairs.
      Sync version of aevaluate_response_strs.
   evaluate_responses(queries: Optional[List[str]] = None, responses: Optional[List[Response]] = None, **eval_kwargs_lists: Dict[str, Any]) -> Dict[str, List[EvaluationResult]]
      Evaluate query, response objs.
      Sync version of aevaluate_responses.
class llama_index.evaluation.CorrectnessEvaluator(service_context: Optional[ServiceContext] = None, eval_template: Optional[Union[BasePromptTemplate, str]] = None, score_threshold: float = 4.0)
   Correctness evaluator.
   Evaluates the correctness of a question answering system. This
   evaluator depends on *reference* answer to be provided, in addition
   to the query string and response string.
   It outputs a score between 1 and 5, where 1 is the worst and 5 is
   the best, along with a reasoning for the score. Passing is defined
   as a score greater than or equal to the given threshold.
   Parameters:
      * **service_context** (*Optional**[**ServiceContext**]*) --

Evaluation

        Service context.
      * **eval_template**
        (*Optional**[**Union**[**BasePromptTemplate**, **str**]**]*)
        -- Template for the evaluation prompt.
      * **score_threshold** (*float*) -- Numerical threshold for
        passing the evaluation, defaults to 4.0.
   async aevaluate(query: Optional[str] = None, response: Optional[str] = None, contexts: Optional[Sequence[str]] = None, reference: Optional[str] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string, retrieved contexts, and
      generated response string.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
   async aevaluate_response(query: Optional[str] = None, response: Optional[Response] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string and generated Response object.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
   evaluate(query: Optional[str] = None, response: Optional[str] = None, contexts: Optional[Sequence[str]] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string, retrieved contexts, and
      generated response string.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
   evaluate_response(query: Optional[str] = None, response: Optional[Response] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string and generated Response object.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
class llama_index.evaluation.DatasetGenerator(nodes: List[BaseNode], service_context: llama_index.indices.service_context.ServiceContext | None = None, num_questions_per_chunk: int = 10, text_question_template: llama_index.prompts.base.BasePromptTemplate | None = None, text_qa_template: llama_index.prompts.base.BasePromptTemplate | None = None, question_gen_query: str | None = None, metadata_mode: MetadataMode = MetadataMode.NONE, show_progress: bool = False)
   Generate dataset (question/ question-answer pairs)     based on the
   given documents.
   NOTE: this is a beta feature, subject to change!
   Parameters:
      * **nodes** (*List**[**Node**]*) -- List of nodes. (Optional)
      * **service_context** (*ServiceContext*) -- Service Context.
      * **num_questions_per_chunk** -- number of question to be
        generated per chunk. Each document is chunked of size 512
        words.
      * **text_question_template** -- Question generation template.
      * **question_gen_query** -- Question generation query.
   async agenerate_dataset_from_nodes(num: int | None = None) -> QueryResponseDataset
      Generates questions for each document.
   async agenerate_questions_from_nodes(num: int | None = None) -> List[str]
      Generates questions for each document.
   classmethod from_documents(documents: List[Document], service_context: llama_index.indices.service_context.ServiceContext | None = None, num_questions_per_chunk: int = 10, text_question_template: llama_index.prompts.base.BasePromptTemplate | None = None, text_qa_template: llama_index.prompts.base.BasePromptTemplate | None = None, question_gen_query: str | None = None, required_keywords: Optional[List[str]] = None, exclude_keywords: Optional[List[str]] = None, show_progress: bool = False) -> DatasetGenerator
      Generate dataset from documents.
   generate_dataset_from_nodes(num: int | None = None) -> QueryResponseDataset
      Generates questions for each document.
   generate_questions_from_nodes(num: int | None = None) -> List[str]
      Generates questions for each document.
pydantic model llama_index.evaluation.EmbeddingQAFinetuneDataset

Evaluation

   Embedding QA Finetuning Dataset.
   Parameters:
      * **queries** (*Dict**[**str**, **str**]*) -- Dict id -> query.
      * **corpus** (*Dict**[**str**, **str**]*) -- Dict id -> string.
      * **relevant_docs** (*Dict**[**str**, **List**[**str**]**]*) --
        Dict query id -> list of doc ids.
      {
         "title": "EmbeddingQAFinetuneDataset",
         "description": "Embedding QA Finetuning Dataset.\n\nArgs:\n    queries (Dict[str, str]): Dict id -> query.\n    corpus (Dict[str, str]): Dict id -> string.\n    relevant_docs (Dict[str, List[str]]): Dict query id -> list of doc ids.",
         "type": "object",
         "properties": {
            "queries": {
               "title": "Queries",
               "type": "object",
               "additionalProperties": {
                  "type": "string"
               }
            },
            "corpus": {
               "title": "Corpus",
               "type": "object",
               "additionalProperties": {
                  "type": "string"
               }
            },
            "relevant_docs": {
               "title": "Relevant Docs",
               "type": "object",
               "additionalProperties": {
                  "type": "array",
                  "items": {
                     "type": "string"
                  }
               }
            }
         },
         "required": [
            "queries",
            "corpus",
            "relevant_docs"
         ]
      }
   Fields:
      * "corpus (Dict[str, str])"
      * "queries (Dict[str, str])"
      * "relevant_docs (Dict[str, List[str]])"
   field corpus: Dict[str, str] [Required]
   field queries: Dict[str, str] [Required]
   field relevant_docs: Dict[str, List[str]] [Required]
   classmethod construct(_fields_set: Optional[SetStr] = None, **values: Any) -> Model
      Creates a new model setting __dict__ and __fields_set__ from
      trusted or pre-validated data. Default values are respected, but
      no other validation is performed. Behaves as if *Config.extra =
      'allow'* was set since it adds all passed values
   copy(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, update: Optional[DictStrAny] = None, deep: bool = False) -> Model
      Duplicate a model, optionally choose which fields to include,
      exclude and change.
      Parameters:
         * **include** -- fields to include in new model
         * **exclude** -- fields to exclude from new model, as with
           values this takes precedence over include
         * **update** -- values to change/add in the new model. Note:
           the data is not validated before creating the new model:
           you should trust this data
         * **deep** -- set to *True* to make a deep copy of the model
      Returns:
         new model instance
   dict(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, by_alias: bool = False, skip_defaults: Optional[bool] = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False) -> DictStrAny
      Generate a dictionary representation of the model, optionally
      specifying which fields to include or exclude.
   classmethod from_json(path: str) -> EmbeddingQAFinetuneDataset

Evaluation

      Load json.
   classmethod from_orm(obj: Any) -> Model
   json(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, by_alias: bool = False, skip_defaults: Optional[bool] = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False, encoder: Optional[Callable[[Any], Any]] = None, models_as_dict: bool = True, **dumps_kwargs: Any) -> unicode
      Generate a JSON representation of the model, *include* and
      *exclude* arguments as per *dict()*.
      *encoder* is an optional function to supply as *default* to
      json.dumps(), other arguments as per *json.dumps()*.
   classmethod parse_file(path: Union[str, Path], *, content_type: unicode = None, encoding: unicode = 'utf8', proto: Protocol = None, allow_pickle: bool = False) -> Model
   classmethod parse_obj(obj: Any) -> Model
   classmethod parse_raw(b: Union[str, bytes], *, content_type: unicode = None, encoding: unicode = 'utf8', proto: Protocol = None, allow_pickle: bool = False) -> Model
   save_json(path: str) -> None
      Save json.
   classmethod schema(by_alias: bool = True, ref_template: unicode = '#/definitions/{model}') -> DictStrAny
   classmethod schema_json(*, by_alias: bool = True, ref_template: unicode = '#/definitions/{model}', **dumps_kwargs: Any) -> unicode
   classmethod update_forward_refs(**localns: Any) -> None
      Try to update ForwardRefs on fields based on this Model,
      globalns and localns.
   classmethod validate(value: Any) -> Model
pydantic model llama_index.evaluation.EvaluationResult
   Evaluation result.
   Output of an BaseEvaluator.
      {
         "title": "EvaluationResult",
         "description": "Evaluation result.\n\nOutput of an BaseEvaluator.",
         "type": "object",
         "properties": {
            "query": {
               "title": "Query",
               "description": "Query string",
               "type": "string"
            },
            "contexts": {
               "title": "Contexts",
               "description": "Context strings",
               "type": "array",
               "items": {
                  "type": "string"
               }
            },
            "response": {
               "title": "Response",
               "description": "Response string",
               "type": "string"
            },
            "passing": {
               "title": "Passing",
               "description": "Binary evaluation result (passing or not)",
               "type": "boolean"
            },
            "feedback": {
               "title": "Feedback",
               "description": "Feedback or reasoning for the response",
               "type": "string"
            },
            "score": {
               "title": "Score",
               "description": "Score for the response",
               "type": "number"
            }
         }
      }
   Fields:
      * "contexts (Optional[Sequence[str]])"
      * "feedback (Optional[str])"
      * "passing (Optional[bool])"
      * "query (Optional[str])"
      * "response (Optional[str])"
      * "score (Optional[float])"
   field contexts: Optional[Sequence[str]] = None
      Context strings
   field feedback: Optional[str] = None
      Feedback or reasoning for the response
   field passing: Optional[bool] = None
      Binary evaluation result (passing or not)
   field query: Optional[str] = None
      Query string

Evaluation

   field response: Optional[str] = None
      Response string
   field score: Optional[float] = None
      Score for the response
   classmethod construct(_fields_set: Optional[SetStr] = None, **values: Any) -> Model
      Creates a new model setting __dict__ and __fields_set__ from
      trusted or pre-validated data. Default values are respected, but
      no other validation is performed. Behaves as if *Config.extra =
      'allow'* was set since it adds all passed values
   copy(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, update: Optional[DictStrAny] = None, deep: bool = False) -> Model
      Duplicate a model, optionally choose which fields to include,
      exclude and change.
      Parameters:
         * **include** -- fields to include in new model
         * **exclude** -- fields to exclude from new model, as with
           values this takes precedence over include
         * **update** -- values to change/add in the new model. Note:
           the data is not validated before creating the new model:
           you should trust this data
         * **deep** -- set to *True* to make a deep copy of the model
      Returns:
         new model instance
   dict(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, by_alias: bool = False, skip_defaults: Optional[bool] = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False) -> DictStrAny
      Generate a dictionary representation of the model, optionally
      specifying which fields to include or exclude.
   classmethod from_orm(obj: Any) -> Model
   json(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, by_alias: bool = False, skip_defaults: Optional[bool] = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False, encoder: Optional[Callable[[Any], Any]] = None, models_as_dict: bool = True, **dumps_kwargs: Any) -> unicode
      Generate a JSON representation of the model, *include* and
      *exclude* arguments as per *dict()*.
      *encoder* is an optional function to supply as *default* to
      json.dumps(), other arguments as per *json.dumps()*.
   classmethod parse_file(path: Union[str, Path], *, content_type: unicode = None, encoding: unicode = 'utf8', proto: Protocol = None, allow_pickle: bool = False) -> Model
   classmethod parse_obj(obj: Any) -> Model
   classmethod parse_raw(b: Union[str, bytes], *, content_type: unicode = None, encoding: unicode = 'utf8', proto: Protocol = None, allow_pickle: bool = False) -> Model
   classmethod schema(by_alias: bool = True, ref_template: unicode = '#/definitions/{model}') -> DictStrAny
   classmethod schema_json(*, by_alias: bool = True, ref_template: unicode = '#/definitions/{model}', **dumps_kwargs: Any) -> unicode
   classmethod update_forward_refs(**localns: Any) -> None
      Try to update ForwardRefs on fields based on this Model,
      globalns and localns.
   classmethod validate(value: Any) -> Model
class llama_index.evaluation.FaithfulnessEvaluator(service_context: llama_index.indices.service_context.ServiceContext | None = None, raise_error: bool = False, eval_template: str | llama_index.prompts.base.BasePromptTemplate | None = None, refine_template: str | llama_index.prompts.base.BasePromptTemplate | None = None)

Evaluation

   Faithfulness evaluator.
   Evaluates whether a response is faithful to the contexts (i.e.
   whether the response is supported by the contexts or hallucinated.)
   This evaluator only considers the response string and the list of
   context strings.
   Parameters:
      * **service_context** (*Optional**[**ServiceContext**]*) -- The
        service context to use for evaluation.
      * **raise_error** (*bool*) -- Whether to raise an error when the
        response is invalid. Defaults to False.
      * **eval_template** (*Optional**[**Union**[**str**,
        **BasePromptTemplate**]**]*) -- The template to use for
        evaluation.
      * **refine_template** (*Optional**[**Union**[**str**,
        **BasePromptTemplate**]**]*) -- The template to use for
        refining the evaluation.
   async aevaluate(query: str | None = None, response: str | None = None, contexts: Optional[Sequence[str]] = None, **kwargs: Any) -> EvaluationResult
      Evaluate whether the response is faithful to the contexts.
   async aevaluate_response(query: Optional[str] = None, response: Optional[Response] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string and generated Response object.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
   evaluate(query: Optional[str] = None, response: Optional[str] = None, contexts: Optional[Sequence[str]] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string, retrieved contexts, and
      generated response string.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
   evaluate_response(query: Optional[str] = None, response: Optional[Response] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string and generated Response object.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
class llama_index.evaluation.GuidelineEvaluator(service_context: Optional[ServiceContext] = None, guidelines: Optional[str] = None, eval_template: Optional[Union[BasePromptTemplate, str]] = None)
   Guideline evaluator.
   Evaluates whether a query and response pair passes the given
   guidelines.
   This evaluator only considers the query string and the response
   string.
   Parameters:
      * **service_context** (*Optional**[**ServiceContext**]*) -- The
        service context to use for evaluation.
      * **guidelines** (*Optional**[**str**]*) -- User-added
        guidelines to use for evaluation. Defaults to None, which uses
        the default guidelines.
      * **eval_template** (*Optional**[**Union**[**str**,
        **BasePromptTemplate**]**]*) -- The template to use for
        evaluation.
   async aevaluate(query: Optional[str] = None, response: Optional[str] = None, contexts: Optional[Sequence[str]] = None, **kwargs: Any) -> EvaluationResult
      Evaluate whether the query and response pair passes the
      guidelines.
   async aevaluate_response(query: Optional[str] = None, response: Optional[Response] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string and generated Response object.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
   evaluate(query: Optional[str] = None, response: Optional[str] = None, contexts: Optional[Sequence[str]] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string, retrieved contexts, and
      generated response string.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
   evaluate_response(query: Optional[str] = None, response: Optional[Response] = None, **kwargs: Any) -> EvaluationResult

Evaluation

      Run evaluation with query string and generated Response object.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
class llama_index.evaluation.HitRate
   Hit rate metric.
   compute(query: Optional[str] = None, expected_ids: Optional[List[str]] = None, retrieved_ids: Optional[List[str]] = None, **kwargs: Any) -> RetrievalMetricResult
      Compute metric.
class llama_index.evaluation.MRR
   MRR metric.
   compute(query: Optional[str] = None, expected_ids: Optional[List[str]] = None, retrieved_ids: Optional[List[str]] = None, **kwargs: Any) -> RetrievalMetricResult
      Compute metric.
class llama_index.evaluation.PairwiseComparisonEvaluator(service_context: Optional[ServiceContext] = None, eval_template: Optional[Union[BasePromptTemplate, str]] = None, enforce_consensus: bool = True)
   Pairwise comparison evaluator.
   Evaluates the quality of a response vs. a "reference" response
   given a question by having an LLM judge which response is better.
   Outputs whether the *response* given is better than the *reference*
   response.
   Parameters:
      * **service_context** (*Optional**[**ServiceContext**]*) -- The
        service context to use for evaluation.
      * **eval_template** (*Optional**[**Union**[**str**,
        **BasePromptTemplate**]**]*) -- The template to use for
        evaluation.
      * **enforce_consensus** (*bool*) -- Whether to enforce consensus
        (consistency if we flip the order of the answers). Defaults to
        True.
   async aevaluate(query: Optional[str] = None, response: Optional[str] = None, contexts: Optional[Sequence[str]] = None, reference: Optional[str] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string, retrieved contexts, and
      generated response string.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
   async aevaluate_response(query: Optional[str] = None, response: Optional[Response] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string and generated Response object.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
   evaluate(query: Optional[str] = None, response: Optional[str] = None, contexts: Optional[Sequence[str]] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string, retrieved contexts, and
      generated response string.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
   evaluate_response(query: Optional[str] = None, response: Optional[Response] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string and generated Response object.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
pydantic model llama_index.evaluation.QueryResponseDataset
   Query Response Dataset.
   The response can be empty if the dataset is generated from
   documents.
   Parameters:
      * **queries** (*Dict**[**str**, **str**]*) -- Query id -> query.
      * **responses** (*Dict**[**str**, **str**]*) -- Query id ->
        response.
      {
         "title": "QueryResponseDataset",
         "description": "Query Response Dataset.\n\nThe response can be empty if the dataset is generated from documents.\n\nArgs:\n    queries (Dict[str, str]): Query id -> query.\n    responses (Dict[str, str]): Query id -> response.",
         "type": "object",
         "properties": {
            "queries": {
               "title": "Queries",
               "description": "Query id -> query",

Evaluation

               "type": "object",
               "additionalProperties": {
                  "type": "string"
               }
            },
            "responses": {
               "title": "Responses",
               "description": "Query id -> response",
               "type": "object",
               "additionalProperties": {
                  "type": "string"
               }
            }
         }
      }
   Fields:
      * "queries (Dict[str, str])"
      * "responses (Dict[str, str])"
   field queries: Dict[str, str] [Optional]
      Query id -> query
   field responses: Dict[str, str] [Optional]
      Query id -> response
   classmethod construct(_fields_set: Optional[SetStr] = None, **values: Any) -> Model
      Creates a new model setting __dict__ and __fields_set__ from
      trusted or pre-validated data. Default values are respected, but
      no other validation is performed. Behaves as if *Config.extra =
      'allow'* was set since it adds all passed values
   copy(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, update: Optional[DictStrAny] = None, deep: bool = False) -> Model
      Duplicate a model, optionally choose which fields to include,
      exclude and change.
      Parameters:
         * **include** -- fields to include in new model
         * **exclude** -- fields to exclude from new model, as with
           values this takes precedence over include
         * **update** -- values to change/add in the new model. Note:
           the data is not validated before creating the new model:
           you should trust this data
         * **deep** -- set to *True* to make a deep copy of the model
      Returns:
         new model instance
   dict(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, by_alias: bool = False, skip_defaults: Optional[bool] = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False) -> DictStrAny
      Generate a dictionary representation of the model, optionally
      specifying which fields to include or exclude.
   classmethod from_json(path: str) -> QueryResponseDataset
      Load json.
   classmethod from_orm(obj: Any) -> Model
   classmethod from_qr_pairs(qr_pairs: List[Tuple[str, str]]) -> QueryResponseDataset
      Create from qr pairs.
   json(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, by_alias: bool = False, skip_defaults: Optional[bool] = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False, encoder: Optional[Callable[[Any], Any]] = None, models_as_dict: bool = True, **dumps_kwargs: Any) -> unicode
      Generate a JSON representation of the model, *include* and
      *exclude* arguments as per *dict()*.
      *encoder* is an optional function to supply as *default* to
      json.dumps(), other arguments as per *json.dumps()*.
   classmethod parse_file(path: Union[str, Path], *, content_type: unicode = None, encoding: unicode = 'utf8', proto: Protocol = None, allow_pickle: bool = False) -> Model
   classmethod parse_obj(obj: Any) -> Model
   classmethod parse_raw(b: Union[str, bytes], *, content_type: unicode = None, encoding: unicode = 'utf8', proto: Protocol = None, allow_pickle: bool = False) -> Model

Evaluation

   save_json(path: str) -> None
      Save json.
   classmethod schema(by_alias: bool = True, ref_template: unicode = '#/definitions/{model}') -> DictStrAny
   classmethod schema_json(*, by_alias: bool = True, ref_template: unicode = '#/definitions/{model}', **dumps_kwargs: Any) -> unicode
   classmethod update_forward_refs(**localns: Any) -> None
      Try to update ForwardRefs on fields based on this Model,
      globalns and localns.
   classmethod validate(value: Any) -> Model
   property qr_pairs: List[Tuple[str, str]]
      Get pairs.
   property questions: List[str]
      Get questions.
llama_index.evaluation.QueryResponseEvaluator
   alias of "RelevancyEvaluator"
class llama_index.evaluation.RelevancyEvaluator(service_context: llama_index.indices.service_context.ServiceContext | None = None, raise_error: bool = False, eval_template: str | llama_index.prompts.base.BasePromptTemplate | None = None, refine_template: str | llama_index.prompts.base.BasePromptTemplate | None = None)
   Relenvancy evaluator.
   Evaluates the relevancy of retrieved contexts and response to a
   query. This evaluator considers the query string, retrieved
   contexts, and response string.
   Parameters:
      * **service_context** (*Optional**[**ServiceContext**]*) -- The
        service context to use for evaluation.
      * **raise_error** (*Optional**[**bool**]*) -- Whether to raise
        an error if the response is invalid. Defaults to False.
      * **eval_template** (*Optional**[**Union**[**str**,
        **BasePromptTemplate**]**]*) -- The template to use for
        evaluation.
      * **refine_template** (*Optional**[**Union**[**str**,
        **BasePromptTemplate**]**]*) -- The template to use for
        refinement.
   async aevaluate(query: str | None = None, response: str | None = None, contexts: Optional[Sequence[str]] = None, **kwargs: Any) -> EvaluationResult
      Evaluate whether the contexts and response are relevant to the
      query.
   async aevaluate_response(query: Optional[str] = None, response: Optional[Response] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string and generated Response object.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
   evaluate(query: Optional[str] = None, response: Optional[str] = None, contexts: Optional[Sequence[str]] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string, retrieved contexts, and
      generated response string.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
   evaluate_response(query: Optional[str] = None, response: Optional[Response] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string and generated Response object.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
llama_index.evaluation.ResponseEvaluator
   alias of "FaithfulnessEvaluator"
pydantic model llama_index.evaluation.RetrievalEvalResult
   Retrieval eval result.
   NOTE: this abstraction might change in the future.
   query
      Query string
      Type:
         str
   expected_ids
      Expected ids
      Type:
         List[str]
   retrieved_ids
      Retrieved ids
      Type:
         List[str]
   metric_dict
      Metric dictionary for the evaluation
      Type:
         Dict[str, BaseRetrievalMetric]
      {
         "title": "RetrievalEvalResult",
         "description": "Retrieval eval result.\n\nNOTE: this abstraction might change in the future.\n\nAttributes:\n    query (str): Query string\n    expected_ids (List[str]): Expected ids\n    retrieved_ids (List[str]): Retrieved ids\n    metric_dict (Dict[str, BaseRetrievalMetric]):             Metric dictionary for the evaluation",

Evaluation

         "type": "object",
         "properties": {
            "query": {
               "title": "Query",
               "description": "Query string",
               "type": "string"
            },
            "expected_ids": {
               "title": "Expected Ids",
               "description": "Expected ids",
               "type": "array",
               "items": {
                  "type": "string"
               }
            },
            "retrieved_ids": {
               "title": "Retrieved Ids",
               "description": "Retrieved ids",
               "type": "array",
               "items": {
                  "type": "string"
               }
            },
            "metric_dict": {
               "title": "Metric Dict",
               "description": "Metric dictionary for the evaluation",
               "type": "object",
               "additionalProperties": {
                  "$ref": "#/definitions/RetrievalMetricResult"
               }
            }
         },
         "required": [
            "query",
            "expected_ids",
            "retrieved_ids",
            "metric_dict"
         ],
         "definitions": {
            "RetrievalMetricResult": {
               "title": "RetrievalMetricResult",
               "description": "Metric result.\n\nAttributes:\n    score (float): Score for the metric\n    metadata (Dict[str, Any]): Metadata for the metric result",
               "type": "object",
               "properties": {
                  "score": {
                     "title": "Score",
                     "description": "Score for the metric",
                     "type": "number"
                  },
                  "metadata": {
                     "title": "Metadata",
                     "description": "Metadata for the metric result",
                     "type": "object"
                  }
               },
               "required": [
                  "score"
               ]
            }
         }
      }
   Config:
      * **arbitrary_types_allowed**: *bool = True*
   Fields:
      * "expected_ids (List[str])"
      * "metric_dict (Dict[str, llama_index.evaluation.retrieval.metr
        ics_base.RetrievalMetricResult])"
      * "query (str)"
      * "retrieved_ids (List[str])"
   field expected_ids: List[str] [Required]
      Expected ids
   field metric_dict: Dict[str, RetrievalMetricResult] [Required]
      Metric dictionary for the evaluation
   field query: str [Required]
      Query string
   field retrieved_ids: List[str] [Required]
      Retrieved ids
   classmethod construct(_fields_set: Optional[SetStr] = None, **values: Any) -> Model
      Creates a new model setting __dict__ and __fields_set__ from
      trusted or pre-validated data. Default values are respected, but
      no other validation is performed. Behaves as if *Config.extra =
      'allow'* was set since it adds all passed values
   copy(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, update: Optional[DictStrAny] = None, deep: bool = False) -> Model
      Duplicate a model, optionally choose which fields to include,
      exclude and change.
      Parameters:
         * **include** -- fields to include in new model
         * **exclude** -- fields to exclude from new model, as with
           values this takes precedence over include
         * **update** -- values to change/add in the new model. Note:
           the data is not validated before creating the new model:
           you should trust this data
         * **deep** -- set to *True* to make a deep copy of the model
      Returns:
         new model instance
   dict(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, by_alias: bool = False, skip_defaults: Optional[bool] = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False) -> DictStrAny

Evaluation

      Generate a dictionary representation of the model, optionally
      specifying which fields to include or exclude.
   classmethod from_orm(obj: Any) -> Model
   json(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, by_alias: bool = False, skip_defaults: Optional[bool] = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False, encoder: Optional[Callable[[Any], Any]] = None, models_as_dict: bool = True, **dumps_kwargs: Any) -> unicode
      Generate a JSON representation of the model, *include* and
      *exclude* arguments as per *dict()*.
      *encoder* is an optional function to supply as *default* to
      json.dumps(), other arguments as per *json.dumps()*.
   classmethod parse_file(path: Union[str, Path], *, content_type: unicode = None, encoding: unicode = 'utf8', proto: Protocol = None, allow_pickle: bool = False) -> Model
   classmethod parse_obj(obj: Any) -> Model
   classmethod parse_raw(b: Union[str, bytes], *, content_type: unicode = None, encoding: unicode = 'utf8', proto: Protocol = None, allow_pickle: bool = False) -> Model
   classmethod schema(by_alias: bool = True, ref_template: unicode = '#/definitions/{model}') -> DictStrAny
   classmethod schema_json(*, by_alias: bool = True, ref_template: unicode = '#/definitions/{model}', **dumps_kwargs: Any) -> unicode
   classmethod update_forward_refs(**localns: Any) -> None
      Try to update ForwardRefs on fields based on this Model,
      globalns and localns.
   classmethod validate(value: Any) -> Model
   property metric_vals_dict: Dict[str, float]
      Dictionary of metric values.
pydantic model llama_index.evaluation.RetrievalMetricResult
   Metric result.
   score
      Score for the metric
      Type:
         float
   metadata
      Metadata for the metric result
      Type:
         Dict[str, Any]
      {
         "title": "RetrievalMetricResult",
         "description": "Metric result.\n\nAttributes:\n    score (float): Score for the metric\n    metadata (Dict[str, Any]): Metadata for the metric result",
         "type": "object",
         "properties": {
            "score": {
               "title": "Score",
               "description": "Score for the metric",
               "type": "number"
            },
            "metadata": {
               "title": "Metadata",
               "description": "Metadata for the metric result",
               "type": "object"
            }
         },
         "required": [
            "score"
         ]
      }
   Fields:
      * "metadata (Dict[str, Any])"
      * "score (float)"
   field metadata: Dict[str, Any] [Optional]
      Metadata for the metric result
   field score: float [Required]
      Score for the metric
   classmethod construct(_fields_set: Optional[SetStr] = None, **values: Any) -> Model
      Creates a new model setting __dict__ and __fields_set__ from
      trusted or pre-validated data. Default values are respected, but
      no other validation is performed. Behaves as if *Config.extra =
      'allow'* was set since it adds all passed values
   copy(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, update: Optional[DictStrAny] = None, deep: bool = False) -> Model

Evaluation

      Duplicate a model, optionally choose which fields to include,
      exclude and change.
      Parameters:
         * **include** -- fields to include in new model
         * **exclude** -- fields to exclude from new model, as with
           values this takes precedence over include
         * **update** -- values to change/add in the new model. Note:
           the data is not validated before creating the new model:
           you should trust this data
         * **deep** -- set to *True* to make a deep copy of the model
      Returns:
         new model instance
   dict(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, by_alias: bool = False, skip_defaults: Optional[bool] = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False) -> DictStrAny
      Generate a dictionary representation of the model, optionally
      specifying which fields to include or exclude.
   classmethod from_orm(obj: Any) -> Model
   json(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, by_alias: bool = False, skip_defaults: Optional[bool] = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False, encoder: Optional[Callable[[Any], Any]] = None, models_as_dict: bool = True, **dumps_kwargs: Any) -> unicode
      Generate a JSON representation of the model, *include* and
      *exclude* arguments as per *dict()*.
      *encoder* is an optional function to supply as *default* to
      json.dumps(), other arguments as per *json.dumps()*.
   classmethod parse_file(path: Union[str, Path], *, content_type: unicode = None, encoding: unicode = 'utf8', proto: Protocol = None, allow_pickle: bool = False) -> Model
   classmethod parse_obj(obj: Any) -> Model
   classmethod parse_raw(b: Union[str, bytes], *, content_type: unicode = None, encoding: unicode = 'utf8', proto: Protocol = None, allow_pickle: bool = False) -> Model
   classmethod schema(by_alias: bool = True, ref_template: unicode = '#/definitions/{model}') -> DictStrAny
   classmethod schema_json(*, by_alias: bool = True, ref_template: unicode = '#/definitions/{model}', **dumps_kwargs: Any) -> unicode
   classmethod update_forward_refs(**localns: Any) -> None
      Try to update ForwardRefs on fields based on this Model,
      globalns and localns.
   classmethod validate(value: Any) -> Model
pydantic model llama_index.evaluation.RetrieverEvaluator
   Retriever evaluator.
   This module will evaluate a retriever using a set of metrics.
   Parameters:
      * **metrics** (*List**[**BaseRetrievalMetric**]*) -- Sequence of
        metrics to evaluate
      * **retriever** -- Retriever to evaluate.
      {
         "title": "RetrieverEvaluator",
         "description": "Retriever evaluator.\n\nThis module will evaluate a retriever using a set of metrics.\n\nArgs:\n    metrics (List[BaseRetrievalMetric]): Sequence of metrics to evaluate\n    retriever: Retriever to evaluate.",
         "type": "object",
         "properties": {
            "metrics": {
               "title": "Metrics"
            },
            "retriever": {
               "title": "Retriever"
            }

Evaluation

         }
      }
   Config:
      * **arbitrary_types_allowed**: *bool = True*
   Fields:
      * "metrics (List[llama_index.evaluation.retrieval.metrics_base.
        BaseRetrievalMetric])"
      * "retriever (llama_index.indices.base_retriever.BaseRetriever)"
   field metrics: List[BaseRetrievalMetric] [Required]
      List of metrics to evaluate
   field retriever: BaseRetriever [Required]
      Retriever to evaluate
   async aevaluate(query: str, expected_ids: List[str], **kwargs: Any) -> RetrievalEvalResult
      Run evaluation with query string, retrieved contexts, and
      generated response string.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
   async aevaluate_dataset(dataset: EmbeddingQAFinetuneDataset, workers: int = 2, show_progress: bool = False, **kwargs: Any) -> List[RetrievalEvalResult]
      Run evaluation with dataset.
   classmethod construct(_fields_set: Optional[SetStr] = None, **values: Any) -> Model
      Creates a new model setting __dict__ and __fields_set__ from
      trusted or pre-validated data. Default values are respected, but
      no other validation is performed. Behaves as if *Config.extra =
      'allow'* was set since it adds all passed values
   copy(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, update: Optional[DictStrAny] = None, deep: bool = False) -> Model
      Duplicate a model, optionally choose which fields to include,
      exclude and change.
      Parameters:
         * **include** -- fields to include in new model
         * **exclude** -- fields to exclude from new model, as with
           values this takes precedence over include
         * **update** -- values to change/add in the new model. Note:
           the data is not validated before creating the new model:
           you should trust this data
         * **deep** -- set to *True* to make a deep copy of the model
      Returns:
         new model instance
   dict(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, by_alias: bool = False, skip_defaults: Optional[bool] = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False) -> DictStrAny
      Generate a dictionary representation of the model, optionally
      specifying which fields to include or exclude.
   evaluate(query: str, expected_ids: List[str], **kwargs: Any) -> RetrievalEvalResult
      Run evaluation results with query string and expected ids.
      Parameters:
         * **query** (*str*) -- Query string
         * **expected_ids** (*List**[**str**]*) -- Expected ids
      Returns:
         Evaluation result
      Return type:
         RetrievalEvalResult
   classmethod from_metric_names(metric_names: List[str], **kwargs: Any) -> BaseRetrievalEvaluator
      Create evaluator from metric names.
      Parameters:
         * **metric_names** (*List**[**str**]*) -- List of metric
           names
         * ****kwargs** -- Additional arguments for the evaluator
   classmethod from_orm(obj: Any) -> Model
   json(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, by_alias: bool = False, skip_defaults: Optional[bool] = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False, encoder: Optional[Callable[[Any], Any]] = None, models_as_dict: bool = True, **dumps_kwargs: Any) -> unicode

Evaluation

      Generate a JSON representation of the model, *include* and
      *exclude* arguments as per *dict()*.
      *encoder* is an optional function to supply as *default* to
      json.dumps(), other arguments as per *json.dumps()*.
   classmethod parse_file(path: Union[str, Path], *, content_type: unicode = None, encoding: unicode = 'utf8', proto: Protocol = None, allow_pickle: bool = False) -> Model
   classmethod parse_obj(obj: Any) -> Model
   classmethod parse_raw(b: Union[str, bytes], *, content_type: unicode = None, encoding: unicode = 'utf8', proto: Protocol = None, allow_pickle: bool = False) -> Model
   classmethod schema(by_alias: bool = True, ref_template: unicode = '#/definitions/{model}') -> DictStrAny
   classmethod schema_json(*, by_alias: bool = True, ref_template: unicode = '#/definitions/{model}', **dumps_kwargs: Any) -> unicode
   classmethod update_forward_refs(**localns: Any) -> None
      Try to update ForwardRefs on fields based on this Model,
      globalns and localns.
   classmethod validate(value: Any) -> Model
class llama_index.evaluation.SemanticSimilarityEvaluator(service_context: Optional[ServiceContext] = None, similarity_fn: Optional[Callable[[...], float]] = None, similarity_mode: Optional[SimilarityMode] = None, similarity_threshold: float = 0.8)
   Embedding similarity evaluator.
   Evaluate the quality of a question answering system by comparing
   the similarity between embeddings of the generated answer and the
   reference answer.
   Inspired by this paper: - Semantic Answer Similarity for Evaluating
   Question Answering Models
      https://arxiv.org/pdf/2108.06130.pdf
   Parameters:
      * **service_context** (*Optional**[**ServiceContext**]*) --
        Service context.
      * **similarity_threshold** (*float*) -- Embedding similarity
        threshold for "passing". Defaults to 0.8.
   async aevaluate(query: Optional[str] = None, response: Optional[str] = None, contexts: Optional[Sequence[str]] = None, reference: Optional[str] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string, retrieved contexts, and
      generated response string.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
   async aevaluate_response(query: Optional[str] = None, response: Optional[Response] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string and generated Response object.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
   evaluate(query: Optional[str] = None, response: Optional[str] = None, contexts: Optional[Sequence[str]] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string, retrieved contexts, and
      generated response string.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
   evaluate_response(query: Optional[str] = None, response: Optional[Response] = None, **kwargs: Any) -> EvaluationResult
      Run evaluation with query string and generated Response object.
      Subclasses can override this method to provide custom evaluation
      logic and take in additional arguments.
llama_index.evaluation.generate_qa_embedding_pairs(nodes: List[TextNode], llm: Optional[LLM] = None, qa_generate_prompt_tmpl: str = 'Context information is below.\n\n---------------------\n{context_str}\n---------------------\n\nGiven the context information and not prior knowledge.\ngenerate only questions based on the below query.\n\nYou are a Teacher/ Professor. Your task is to setup {num_questions_per_chunk} questions for an upcoming quiz/examination. The questions should be diverse in nature across the document. Restrict the questions to the context information provided."\n', num_questions_per_chunk: int = 2) -> EmbeddingQAFinetuneDataset

Evaluation

   Generate examples given a set of nodes.
llama_index.evaluation.generate_question_context_pairs(nodes: List[TextNode], llm: Optional[LLM] = None, qa_generate_prompt_tmpl: str = 'Context information is below.\n\n---------------------\n{context_str}\n---------------------\n\nGiven the context information and not prior knowledge.\ngenerate only questions based on the below query.\n\nYou are a Teacher/ Professor. Your task is to setup {num_questions_per_chunk} questions for an upcoming quiz/examination. The questions should be diverse in nature across the document. Restrict the questions to the context information provided."\n', num_questions_per_chunk: int = 2) -> EmbeddingQAFinetuneDataset
   Generate examples given a set of nodes.
llama_index.evaluation.get_retrieval_results_df(names: List[str], results_arr: List[List[RetrievalEvalResult]], metric_keys: Optional[List[str]] = None) -> DataFrame
   Display retrieval results.
llama_index.evaluation.resolve_metrics(metrics: List[str]) -> List[BaseRetrievalMetric]
   Resolve metrics from list of metric names.


Querying an Index

This doc shows the classes that are used to query indices.
Main Query Classes
Querying an index involves a three main components:
* **Retrievers**: A retriever class retrieves a set of Nodes from an
  index given a query.
* **Response Synthesizer**: This class takes in a set of Nodes and
  synthesizes an answer given a query.
* **Query Engine**: This class takes in a query and returns a Response
  object. It can make use of Retrievers and Response Synthesizer
  modules under the hood.
* **Chat Engines**: This class enables conversation over a knowledge
  base. It is the stateful version of a query engine that keeps track
  of conversation history.
Main query classes
^^^^^^^^^^^^^^^^^^
* Retrievers
* Response Synthesizer
* Query Engines
* Chat Engines
Additional Query Classes
We also detail some additional query classes below.
* **Query Bundle**: This is the input to the query classes: retriever,
  response synthesizer,
     and query engine. It enables the user to customize the string(s)
     used for embedding-based query.
* **Query Transform**: This class augments a raw query string with
     associated transformations to improve index querying. Can be used
     with a Retriever (see TransformRetriever) or QueryEngine.
Additional query classes
^^^^^^^^^^^^^^^^^^^^^^^^
* Query Bundle
* Query Transform


Example Notebooks

We offer a wide variety of example notebooks. They are referenced
throughout the documentation.
Example notebooks are found here.


Indices

This doc shows both the overarching class used to represent an index.
These classes allow for index creation, insertion, and also querying.
We first show the different index subclasses. We then show the base
class that all indices inherit from, which contains parameters and
methods common to all indices.
Index Data Structures
^^^^^^^^^^^^^^^^^^^^^
* Summary Index
* Table Index
* Tree Index
* Vector Store Index
* Structured Store Index
* Knowledge Graph Index
* Empty Index
Base Index Class
Base index classes.
llama_index.indices.base.BaseGPTIndex
   alias of "BaseIndex"
class llama_index.indices.base.BaseIndex(nodes: Optional[Sequence[BaseNode]] = None, index_struct: Optional[IS] = None, storage_context: Optional[StorageContext] = None, service_context: Optional[ServiceContext] = None, show_progress: bool = False, **kwargs: Any)
   Base LlamaIndex.
   Parameters:
      * **nodes** (*List**[**Node**]*) -- List of nodes to index
      * **show_progress** (*bool*) -- Whether to show tqdm progress
        bars. Defaults to False.
      * **service_context** (*ServiceContext*) -- Service context
        container (contains components like LLMPredictor,
        PromptHelper, etc.).
   build_index_from_nodes(nodes: Sequence[BaseNode]) -> IS
      Build the index from nodes.
   delete(doc_id: str, **delete_kwargs: Any) -> None
      Delete a document from the index. All nodes in the index related
      to the index will be deleted.
      Parameters:
         **doc_id** (*str*) -- A doc_id of the ingested document
   delete_nodes(node_ids: List[str], delete_from_docstore: bool = False, **delete_kwargs: Any) -> None
      Delete a list of nodes from the index.
      Parameters:
         **doc_ids** (*List**[**str**]*) -- A list of doc_ids from the
         nodes to delete
   delete_ref_doc(ref_doc_id: str, delete_from_docstore: bool = False, **delete_kwargs: Any) -> None
      Delete a document and it's nodes by using ref_doc_id.
   property docstore: BaseDocumentStore
      Get the docstore corresponding to the index.
   classmethod from_documents(documents: Sequence[Document], storage_context: Optional[StorageContext] = None, service_context: Optional[ServiceContext] = None, show_progress: bool = False, **kwargs: Any) -> IndexType
      Create index from documents.
      Parameters:
         **documents**
         (*Optional**[**Sequence**[**BaseDocument**]**]*) -- List of
         documents to build the index from.
   property index_id: str
      Get the index struct.
   property index_struct: IS
      Get the index struct.
   insert(document: Document, **insert_kwargs: Any) -> None
      Insert a document.
   insert_nodes(nodes: Sequence[BaseNode], **insert_kwargs: Any) -> None
      Insert nodes.
   abstract property ref_doc_info: Dict[str, RefDocInfo]
      Retrieve a dict mapping of ingested documents and their
      nodes+metadata.
   refresh(documents: Sequence[Document], **update_kwargs: Any) -> List[bool]
      Refresh an index with documents that have changed.
      This allows users to save LLM and Embedding model calls, while
      only updating documents that have any changes in text or
      metadata. It will also insert any documents that previously were
      not stored.
   refresh_ref_docs(documents: Sequence[Document], **update_kwargs: Any) -> List[bool]
      Refresh an index with documents that have changed.
      This allows users to save LLM and Embedding model calls, while
      only updating documents that have any changes in text or

Indices

      metadata. It will also insert any documents that previously were
      not stored.
   set_index_id(index_id: str) -> None
      Set the index id.
      NOTE: if you decide to set the index_id on the index_struct
      manually, you will need to explicitly call *add_index_struct* on
      the *index_store* to update the index store.
      Parameters:
         **index_id** (*str*) -- Index id to set.
   update(document: Document, **update_kwargs: Any) -> None
      Update a document and it's corresponding nodes.
      This is equivalent to deleting the document and then inserting
      it again.
      Parameters:
         * **document** (*Union**[**BaseDocument**, **BaseIndex**]*)
           -- document to update
         * **insert_kwargs** (*Dict*) -- kwargs to pass to insert
         * **delete_kwargs** (*Dict*) -- kwargs to pass to delete
   update_ref_doc(document: Document, **update_kwargs: Any) -> None
      Update a document and it's corresponding nodes.
      This is equivalent to deleting the document and then inserting
      it again.
      Parameters:
         * **document** (*Union**[**BaseDocument**, **BaseIndex**]*)
           -- document to update
         * **insert_kwargs** (*Dict*) -- kwargs to pass to insert
         * **delete_kwargs** (*Dict*) -- kwargs to pass to delete


Memory

pydantic model llama_index.memory.BaseMemory
   Base class for all memory types.
   NOTE: The interface for memory is not yet finalized and is subject
   to change.
      {
         "title": "BaseMemory",
         "description": "Base class for all memory types.\n\nNOTE: The interface for memory is not yet finalized and is subject to change.",
         "type": "object",
         "properties": {}
      }
   classmethod construct(_fields_set: Optional[SetStr] = None, **values: Any) -> Model
      Creates a new model setting __dict__ and __fields_set__ from
      trusted or pre-validated data. Default values are respected, but
      no other validation is performed. Behaves as if *Config.extra =
      'allow'* was set since it adds all passed values
   copy(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, update: Optional[DictStrAny] = None, deep: bool = False) -> Model
      Duplicate a model, optionally choose which fields to include,
      exclude and change.
      Parameters:
         * **include** -- fields to include in new model
         * **exclude** -- fields to exclude from new model, as with
           values this takes precedence over include
         * **update** -- values to change/add in the new model. Note:
           the data is not validated before creating the new model:
           you should trust this data
         * **deep** -- set to *True* to make a deep copy of the model
      Returns:
         new model instance
   dict(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, by_alias: bool = False, skip_defaults: Optional[bool] = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False) -> DictStrAny
      Generate a dictionary representation of the model, optionally
      specifying which fields to include or exclude.
   abstract classmethod from_defaults(chat_history: Optional[List[ChatMessage]] = None, llm: Optional[LLM] = None) -> BaseMemory
      Create a chat memory from defaults.
   classmethod from_orm(obj: Any) -> Model
   abstract get() -> List[ChatMessage]
      Get chat history.
   abstract get_all() -> List[ChatMessage]
      Get all chat history.
   json(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, by_alias: bool = False, skip_defaults: Optional[bool] = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False, encoder: Optional[Callable[[Any], Any]] = None, models_as_dict: bool = True, **dumps_kwargs: Any) -> unicode
      Generate a JSON representation of the model, *include* and
      *exclude* arguments as per *dict()*.
      *encoder* is an optional function to supply as *default* to
      json.dumps(), other arguments as per *json.dumps()*.
   classmethod parse_file(path: Union[str, Path], *, content_type: unicode = None, encoding: unicode = 'utf8', proto: Protocol = None, allow_pickle: bool = False) -> Model
   classmethod parse_obj(obj: Any) -> Model
   classmethod parse_raw(b: Union[str, bytes], *, content_type: unicode = None, encoding: unicode = 'utf8', proto: Protocol = None, allow_pickle: bool = False) -> Model

Memory

   abstract put(message: ChatMessage) -> None
      Put chat history.
   abstract reset() -> None
      Reset chat history.
   classmethod schema(by_alias: bool = True, ref_template: unicode = '#/definitions/{model}') -> DictStrAny
   classmethod schema_json(*, by_alias: bool = True, ref_template: unicode = '#/definitions/{model}', **dumps_kwargs: Any) -> unicode
   abstract set(messages: List[ChatMessage]) -> None
      Set chat history.
   classmethod update_forward_refs(**localns: Any) -> None
      Try to update ForwardRefs on fields based on this Model,
      globalns and localns.
   classmethod validate(value: Any) -> Model
pydantic model llama_index.memory.ChatMemoryBuffer
   Simple buffer for storing chat history.
      {
         "title": "ChatMemoryBuffer",
         "description": "Simple buffer for storing chat history.",
         "type": "object",
         "properties": {
            "token_limit": {
               "title": "Token Limit",
               "type": "integer"
            },
            "chat_history": {
               "title": "Chat History",
               "type": "array",
               "items": {
                  "$ref": "#/definitions/ChatMessage"
               }
            }
         },
         "required": [
            "token_limit"
         ],
         "definitions": {
            "MessageRole": {
               "title": "MessageRole",
               "description": "Message role.",
               "enum": [
                  "system",
                  "user",
                  "assistant",
                  "function"
               ],
               "type": "string"
            },
            "ChatMessage": {
               "title": "ChatMessage",
               "description": "Chat message.",
               "type": "object",
               "properties": {
                  "role": {
                     "default": "user",
                     "allOf": [
                        {
                           "$ref": "#/definitions/MessageRole"
                        }
                     ]
                  },
                  "content": {
                     "title": "Content",
                     "default": "",
                     "type": "string"
                  },
                  "additional_kwargs": {
                     "title": "Additional Kwargs",
                     "type": "object"
                  }
               }
            }
         }
      }
   Fields:
      * "chat_history (List[llama_index.llms.base.ChatMessage])"
      * "token_limit (int)"
      * "tokenizer_fn (Callable[[str], List])"
   field chat_history: List[ChatMessage] [Optional]
      Validated by:
         * "validate_memory"
   field token_limit: int [Required]
      Validated by:
         * "validate_memory"
   field tokenizer_fn: Callable[[str], List] [Optional]
      Validated by:
         * "validate_memory"
   classmethod construct(_fields_set: Optional[SetStr] = None, **values: Any) -> Model
      Creates a new model setting __dict__ and __fields_set__ from
      trusted or pre-validated data. Default values are respected, but
      no other validation is performed. Behaves as if *Config.extra =
      'allow'* was set since it adds all passed values
   copy(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, update: Optional[DictStrAny] = None, deep: bool = False) -> Model
      Duplicate a model, optionally choose which fields to include,
      exclude and change.
      Parameters:
         * **include** -- fields to include in new model
         * **exclude** -- fields to exclude from new model, as with
           values this takes precedence over include
         * **update** -- values to change/add in the new model. Note:

Memory

           the data is not validated before creating the new model:
           you should trust this data
         * **deep** -- set to *True* to make a deep copy of the model
      Returns:
         new model instance
   dict(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, by_alias: bool = False, skip_defaults: Optional[bool] = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False) -> DictStrAny
      Generate a dictionary representation of the model, optionally
      specifying which fields to include or exclude.
   classmethod from_defaults(chat_history: Optional[List[ChatMessage]] = None, llm: Optional[LLM] = None, token_limit: Optional[int] = None, tokenizer_fn: Optional[Callable[[str], List]] = None) -> ChatMemoryBuffer
      Create a chat memory buffer from an LLM.
   classmethod from_dict(json_dict: dict) -> ChatMemoryBuffer
   classmethod from_orm(obj: Any) -> Model
   classmethod from_string(json_str: str) -> ChatMemoryBuffer
   get() -> List[ChatMessage]
      Get chat history.
   get_all() -> List[ChatMessage]
      Get all chat history.
   json(*, include: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, exclude: Optional[Union[AbstractSetIntStr, MappingIntStrAny]] = None, by_alias: bool = False, skip_defaults: Optional[bool] = None, exclude_unset: bool = False, exclude_defaults: bool = False, exclude_none: bool = False, encoder: Optional[Callable[[Any], Any]] = None, models_as_dict: bool = True, **dumps_kwargs: Any) -> unicode
      Generate a JSON representation of the model, *include* and
      *exclude* arguments as per *dict()*.
      *encoder* is an optional function to supply as *default* to
      json.dumps(), other arguments as per *json.dumps()*.
   classmethod parse_file(path: Union[str, Path], *, content_type: unicode = None, encoding: unicode = 'utf8', proto: Protocol = None, allow_pickle: bool = False) -> Model
   classmethod parse_obj(obj: Any) -> Model
   classmethod parse_raw(b: Union[str, bytes], *, content_type: unicode = None, encoding: unicode = 'utf8', proto: Protocol = None, allow_pickle: bool = False) -> Model
   put(message: ChatMessage) -> None
      Put chat history.
   reset() -> None
      Reset chat history.
   classmethod schema(by_alias: bool = True, ref_template: unicode = '#/definitions/{model}') -> DictStrAny
   classmethod schema_json(*, by_alias: bool = True, ref_template: unicode = '#/definitions/{model}', **dumps_kwargs: Any) -> unicode
   set(messages: List[ChatMessage]) -> None
      Set chat history.
   to_dict() -> dict
      Convert memory to dict.
   to_string() -> str
      Convert memory to string.
   classmethod update_forward_refs(**localns: Any) -> None
      Try to update ForwardRefs on fields based on this Model,
      globalns and localns.
   classmethod validate(value: Any) -> Model
   validator validate_memory  »  *all fields*