Repository: stanfordnlp/dspy
Pull request #8417 (2025-06-18 19:48:18)

Add gemini to LM provider
Gemini has been irrelevant until the recent gemini-2.5 launch. I have been seeing lots of Gemini users now, so we should include it in the LM setup. 

To be fair, gemini-2.5 is pretty good, definitely not a marketing thing. 

change preview:

<img width="971" alt="image" src="https://github.com/user-attachments/assets/537d6e04-0494-4c18-8191-ee46a191fd8e" />



Patch:
diff --git a/.gitignore b/.gitignore
index 9541a1c6..14f48d7f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -58,6 +58,7 @@ assertion.log
 /.devcontainer/.personalization.sh
 
 .mypy_cache
+CLAUDE.local.md
 dummy.csv
 docs/docs/**/*.json*
 *.index
diff --git a/docs/docs/index.md b/docs/docs/index.md
index e157bd56..2631319f 100644
--- a/docs/docs/index.md
+++ b/docs/docs/index.md
@@ -23,7 +23,7 @@ Instead of wrangling prompts or training jobs, DSPy (Declarative Self-improving
 !!! info "Getting Started I: Install DSPy and set up your LM"
 
     ```bash
-    > pip install -U dspy datasets
+    > pip install -U dspy
     ```
 
     === "OpenAI"
@@ -53,6 +53,15 @@ Instead of wrangling prompts or training jobs, DSPy (Declarative Self-improving
         dspy.configure(lm=lm)
         ```
 
+    === "Gemini"
+        You can authenticate by setting the GEMINI_API_KEY env variable or passing `api_key` below.
+
+        ```python linenums="1"
+        import dspy
+        lm = dspy.LM('gemini/gemini-2.5-flash', api_key='YOUR_GEMINI_API_KEY')
+        dspy.configure(lm=lm)
+        ```
+
     === "Local LMs on your laptop"
           First, install [Ollama](https://github.com/ollama/ollama) and launch its server with your LM.
 
@@ -313,7 +322,13 @@ Given a few tens or hundreds of representative _inputs_ of your task and a _metr
 !!! info "Getting Started III: Optimizing the LM prompts or weights in DSPy programs"
     A typical simple optimization run costs on the order of $2 USD and takes around 20 minutes, but be careful when running optimizers with very large LMs or very large datasets.
     Optimization can cost as little as a few cents or up to tens of dollars, depending on your LM, dataset, and configuration.
-    
+
+    Examples below rely on HuggingFace/datasets, you can install it by the command below.
+
+    ```bash
+    > pip install -U datasets
+    ```
+
     === "Optimizing prompts for a ReAct agent"
         This is a minimal but fully runnable example of setting up a `dspy.ReAct` agent that answers questions via
         search from Wikipedia and then optimizing it using `dspy.MIPROv2` in the cheap `light` mode on 500
diff --git a/docs/docs/tutorials/agents/index.ipynb b/docs/docs/tutorials/agents/index.ipynb
index 493ce4a0..d2ce3c71 100644
--- a/docs/docs/tutorials/agents/index.ipynb
+++ b/docs/docs/tutorials/agents/index.ipynb
@@ -500,23 +500,20 @@
     "        metric=top5_recall,\n",
     "        num_threads=16,\n",
     "        display_progress=True,\n",
-    "        # To record the outputs and detailed scores to MLflow\n",
-    "        return_all_scores=True,\n",
-    "        return_outputs=True,\n",
     "    )\n",
     "\n",
     "    # Evaluate the program as usual\n",
-    "    aggregated_score, outputs, all_scores = evaluate(cot)\n",
+    "    result = evaluate(cot)\n",
     "\n",
     "    # Log the aggregated score\n",
-    "    mlflow.log_metric(\"top5_recall\", aggregated_score)\n",
+    "    mlflow.log_metric(\"top5_recall\", result.score)\n",
     "    # Log the detailed evaluation results as a table\n",
     "    mlflow.log_table(\n",
     "        {\n",
     "            \"Claim\": [example.claim for example in eval_set],\n",
     "            \"Expected Titles\": [example.titles for example in eval_set],\n",
-    "            \"Predicted Titles\": outputs,\n",
-    "            \"Top 5 Recall\": all_scores,\n",
+    "            \"Predicted Titles\": [output[1] for output in result.results],\n",
+    "            \"Top 5 Recall\": [output[2] for output in result.results],\n",
     "        },\n",
     "        artifact_file=\"eval_results.json\",\n",
     "    )\n",
diff --git a/docs/docs/tutorials/classification_finetuning/index.ipynb b/docs/docs/tutorials/classification_finetuning/index.ipynb
index b318e472..80f949a4 100644
--- a/docs/docs/tutorials/classification_finetuning/index.ipynb
+++ b/docs/docs/tutorials/classification_finetuning/index.ipynb
@@ -568,23 +568,20 @@
     "        metric=extraction_correctness_metric,\n",
     "        num_threads=16,\n",
     "        display_progress=True,\n",
-    "        # To record the outputs and detailed scores to MLflow\n",
-    "        return_all_scores=True,\n",
-    "        return_outputs=True,\n",
     "    )\n",
     "\n",
     "    # Evaluate the program as usual\n",
-    "    aggregated_score, outputs, all_scores = evaluate_correctness(people_extractor)\n",
+    "    result = evaluate_correctness(people_extractor)\n",
     "\n",
     "    # Log the aggregated score\n",
-    "    mlflow.log_metric(\"exact_match\", aggregated_score)\n",
+    "    mlflow.log_metric(\"exact_match\", result.score)\n",
     "    # Log the detailed evaluation results as a table\n",
     "    mlflow.log_table(\n",
     "        {\n",
     "            \"Text\": [example.text for example in devset],\n",
     "            \"Expected\": [example.example_label for example in devset],\n",
-    "            \"Predicted\": outputs,\n",
-    "            \"Exact match\": all_scores,\n",
+    "            \"Predicted\": [output[1] for output in result.results],\n",
+    "            \"Exact match\": [output[2] for output in result.results],\n",
     "        },\n",
     "        artifact_file=\"eval_results.json\",\n",
     "    )\n",
diff --git a/docs/docs/tutorials/entity_extraction/index.ipynb b/docs/docs/tutorials/entity_extraction/index.ipynb
index d67bfc19..f948e9ef 100644
--- a/docs/docs/tutorials/entity_extraction/index.ipynb
+++ b/docs/docs/tutorials/entity_extraction/index.ipynb
@@ -514,23 +514,20 @@
         "        metric=extraction_correctness_metric,\n",
         "        num_threads=24,\n",
         "        display_progress=True,\n",
-        "        # To record the outputs and detailed scores to MLflow\n",
-        "        return_all_scores=True,\n",
-        "        return_outputs=True,\n",
         "    )\n",
         "\n",
         "    # Evaluate the program as usual\n",
-        "    aggregated_score, outputs, all_scores = evaluate_correctness(people_extractor)\n",
+        "    result = evaluate_correctness(people_extractor)\n",
         "\n",
         "    # Log the aggregated score\n",
-        "    mlflow.log_metric(\"exact_match\", aggregated_score)\n",
+        "    mlflow.log_metric(\"exact_match\", result.score)\n",
         "    # Log the detailed evaluation results as a table\n",
         "    mlflow.log_table(\n",
         "        {\n",
         "            \"Tokens\": [example.tokens for example in test_set],\n",
         "            \"Expected\": [example.expected_extracted_people for example in test_set],\n",
-        "            \"Predicted\": outputs,\n",
-        "            \"Exact match\": all_scores,\n",
+        "            \"Predicted\": [output[1] for output in result.results],\n",
+        "            \"Exact match\": [output[2] for output in result.results],\n",
         "        },\n",
         "        artifact_file=\"eval_results.json\",\n",
         "    )\n",
diff --git a/docs/docs/tutorials/math/index.ipynb b/docs/docs/tutorials/math/index.ipynb
index a0f70065..10de528d 100644
--- a/docs/docs/tutorials/math/index.ipynb
+++ b/docs/docs/tutorials/math/index.ipynb
@@ -369,21 +369,21 @@
     "\n",
     "# Start an MLflow Run to record the evaluation\n",
     "with mlflow.start_run(run_name=\"math_evaluation\"):\n",
-    "    kwargs = dict(num_threads=THREADS, display_progress=True, return_all_scores=True, return_outputs=True)\n",
+    "    kwargs = dict(num_threads=THREADS, display_progress=True)\n",
     "    evaluate = dspy.Evaluate(devset=dataset.dev, metric=dataset.metric, **kwargs)\n",
     "\n",
     "    # Evaluate the program as usual\n",
-    "    aggregated_score, outputs, all_scores = evaluate(module)\n",
+    "    result = evaluate(module)\n",
     "\n",
     "    # Log the aggregated score\n",
-    "    mlflow.log_metric(\"correctness\", aggregated_score)\n",
+    "    mlflow.log_metric(\"correctness\", result.score)\n",
     "    # Log the detailed evaluation results as a table\n",
     "    mlflow.log_table(\n",
     "        {\n",
     "            \"Question\": [example.question for example in dataset.dev],\n",
     "            \"Gold Answer\": [example.answer for example in dataset.dev],\n",
-    "            \"Predicted Answer\": outputs,\n",
-    "            \"Correctness\": all_scores,\n",
+    "            \"Predicted Answer\": [output[1] for output in result.results],\n",
+    "            \"Correctness\": [output[2] for output in result.results],\n",
     "        },\n",
     "        artifact_file=\"eval_results.json\",\n",
     "    )\n",
diff --git a/docs/docs/tutorials/multihop_search/index.ipynb b/docs/docs/tutorials/multihop_search/index.ipynb
index b769b02e..00ed91e8 100644
--- a/docs/docs/tutorials/multihop_search/index.ipynb
+++ b/docs/docs/tutorials/multihop_search/index.ipynb
@@ -534,23 +534,20 @@
     "        metric=top5_recall,\n",
     "        num_threads=16,\n",
     "        display_progress=True,\n",
-    "        # To record the outputs and detailed scores to MLflow\n",
-    "        return_all_scores=True,\n",
-    "        return_outputs=True,\n",
     "    )\n",
     "\n",
     "    # Evaluate the program as usual\n",
-    "    aggregated_score, outputs, all_scores = evaluate(Hop())\n",
+    "    result = evaluate(Hop())\n",
     "\n",
     "    # Log the aggregated score\n",
-    "    mlflow.log_metric(\"top5_recall\", aggregated_score)\n",
+    "    mlflow.log_metric(\"top5_recall\", result.score)\n",
     "    # Log the detailed evaluation results as a table\n",
     "    mlflow.log_table(\n",
     "        {\n",
     "            \"Claim\": [example.claim for example in eval_set],\n",
     "            \"Expected Titles\": [example.titles for example in eval_set],\n",
-    "            \"Predicted Titles\": outputs,\n",
-    "            \"Top 5 Recall\": all_scores,\n",
+    "            \"Predicted Titles\": [output[1] for output in result.results],\n",
+    "            \"Top 5 Recall\": [output[2] for output in result.results],\n",
     "        },\n",
     "        artifact_file=\"eval_results.json\",\n",
     "    )\n",
diff --git a/docs/docs/tutorials/rag/index.ipynb b/docs/docs/tutorials/rag/index.ipynb
index b3d29e83..0f4a241a 100644
--- a/docs/docs/tutorials/rag/index.ipynb
+++ b/docs/docs/tutorials/rag/index.ipynb
@@ -731,24 +731,21 @@
     "        metric=metric,\n",
     "        num_threads=24,\n",
     "        display_progress=True,\n",
-    "        # To record the outputs and detailed scores to MLflow\n",
-    "        return_all_scores=True,\n",
-    "        return_outputs=True,\n",
     "    )\n",
     "\n",
     "    # Evaluate the program as usual\n",
-    "    aggregated_score, outputs, all_scores = evaluate(cot)\n",
+    "    result = evaluate(cot)\n",
     "\n",
     "\n",
     "    # Log the aggregated score\n",
-    "    mlflow.log_metric(\"semantic_f1_score\", aggregated_score)\n",
+    "    mlflow.log_metric(\"semantic_f1_score\", result.score)\n",
     "    # Log the detailed evaluation results as a table\n",
     "    mlflow.log_table(\n",
     "        {\n",
     "            \"Question\": [example.question for example in eval_set],\n",
     "            \"Gold Response\": [example.response for example in eval_set],\n",
-    "            \"Predicted Response\": outputs,\n",
-    "            \"Semantic F1 Score\": all_scores,\n",
+    "            \"Predicted Response\": [output[1] for output in result.results],\n",
+    "            \"Semantic F1 Score\": [output[2] for output in result.results],\n",
     "        },\n",
     "        artifact_file=\"eval_results.json\",\n",
     "    )\n",
@@ -1471,6 +1468,11 @@
     "\n",
     "The first step is to look at your system outputs, which will allow you to identify the sources of lower performance if any. While doing all of this, make sure you continue to refine your metric, e.g. by optimizing against your judgments, and to collect more (or more realistic) data, e.g. from related domains or from putting a demo of your system in front of users."
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": []
   }
  ],
  "metadata": {
diff --git a/dspy/clients/cache.py b/dspy/clients/cache.py
index bfa7dbde..20b50735 100644
--- a/dspy/clients/cache.py
+++ b/dspy/clients/cache.py
@@ -30,7 +30,6 @@ class Cache:
         disk_cache_dir: str,
         disk_size_limit_bytes: Optional[int] = 1024 * 1024 * 10,
         memory_max_entries: Optional[int] = 1000000,
-
     ):
         """
         Args:
@@ -230,9 +229,11 @@ def request_cache(
                 return cached_result
 
             # Otherwise, compute and store the result
+            # Make a copy of the original request in case it's modified in place, e.g., deleting some fields
+            original_request = copy.deepcopy(modified_request)
             result = fn(*args, **kwargs)
             # `enable_memory_cache` can be provided at call time to avoid indefinite growth.
-            cache.put(modified_request, result, ignored_args_for_cache_key, enable_memory_cache)
+            cache.put(original_request, result, ignored_args_for_cache_key, enable_memory_cache)
 
             return result
 
@@ -249,8 +250,10 @@ def request_cache(
                 return cached_result
 
             # Otherwise, compute and store the result
+            # Make a copy of the original request in case it's modified in place, e.g., deleting some fields
+            original_request = copy.deepcopy(modified_request)
             result = await fn(*args, **kwargs)
-            cache.put(modified_request, result, ignored_args_for_cache_key, enable_memory_cache)
+            cache.put(original_request, result, ignored_args_for_cache_key, enable_memory_cache)
 
             return result
 
diff --git a/dspy/evaluate/evaluate.py b/dspy/evaluate/evaluate.py
index ea86c0c0..ee05ff7d 100644
--- a/dspy/evaluate/evaluate.py
+++ b/dspy/evaluate/evaluate.py
@@ -9,6 +9,7 @@ if TYPE_CHECKING:
 import tqdm
 
 import dspy
+from dspy.primitives.prediction import Prediction
 from dspy.utils.callback import with_callbacks
 from dspy.utils.parallelizer import ParallelExecutor
 
@@ -42,6 +43,21 @@ except ImportError:
 logger = logging.getLogger(__name__)
 
 
+class EvaluationResult(Prediction):
+    """
+    A class that represents the result of an evaluation.
+    It is a subclass of `dspy.Prediction` that contains the following fields
+
+    - score: An float value (e.g., 67.30) representing the overall performance
+    - results: a list of (example, prediction, score) tuples for each example in devset
+    """
+    def __init__(self, score: float, results: list[Tuple["dspy.Example", "dspy.Example", Any]]):
+        super().__init__(score=score, results=results)
+
+    def __repr__(self):
+        return f"EvaluationResult(score={self.score}, results=<list of {len(self.results)} results>)"
+
+
 class Evaluate:
     """DSPy Evaluate class.
 
@@ -58,8 +74,6 @@ class Evaluate:
         display_progress: bool = False,
         display_table: Union[bool, int] = False,
         max_errors: Optional[int] = None,
-        return_all_scores: bool = False,
-        return_outputs: bool = False,
         provide_traceback: Optional[bool] = None,
         failure_score: float = 0.0,
         **kwargs,
@@ -74,8 +88,6 @@ class Evaluate:
                 If a number is passed, the evaluation results will be truncated to that number before displayed.
             max_errors (Optional[int]): The maximum number of errors to allow before
                 stopping evaluation. If ``None``, inherits from ``dspy.settings.max_errors``.
-            return_all_scores (bool): Whether to return scores for every data record in `devset`.
-            return_outputs (bool): Whether to return the dspy program's outputs for every data in `devset`.
             provide_traceback (Optional[bool]): Whether to provide traceback information during evaluation.
             failure_score (float): The default score to use if evaluation fails due to an exception.
         """
@@ -85,8 +97,6 @@ class Evaluate:
         self.display_progress = display_progress
         self.display_table = display_table
         self.max_errors = max_errors
-        self.return_all_scores = return_all_scores
-        self.return_outputs = return_outputs
         self.provide_traceback = provide_traceback
         self.failure_score = failure_score
 
@@ -99,10 +109,8 @@ class Evaluate:
         num_threads: Optional[int] = None,
         display_progress: Optional[bool] = None,
         display_table: Optional[Union[bool, int]] = None,
-        return_all_scores: Optional[bool] = None,
-        return_outputs: Optional[bool] = None,
         callback_metadata: Optional[dict[str, Any]] = None,
-    ):
+    ) -> EvaluationResult:
         """
         Args:
             program (dspy.Module): The DSPy program to evaluate.
@@ -114,36 +122,20 @@ class Evaluate:
                 `self.display_progress`.
             display_table (Union[bool, int]): Whether to display the evaluation results in a table. if not provided, use
                 `self.display_table`. If a number is passed, the evaluation results will be truncated to that number before displayed.
-            return_all_scores (bool): Whether to return scores for every data record in `devset`. if not provided,
-                use `self.return_all_scores`.
-            return_outputs (bool): Whether to return the dspy program's outputs for every data in `devset`. if not
-                provided, use `self.return_outputs`.
             callback_metadata (dict): Metadata to be used for evaluate callback handlers.
 
         Returns:
-            The evaluation results are returned in different formats based on the flags:
-
-            - Base return: A float percentage score (e.g., 67.30) representing overall performance
-
-            - With `return_all_scores=True`:
-                Returns (overall_score, individual_scores) where individual_scores is a list of
-                float scores for each example in devset
-
-            - With `return_outputs=True`:
-                Returns (overall_score, result_triples) where result_triples is a list of
-                (example, prediction, score) tuples for each example in devset
-
-            - With both flags=True:
-                Returns (overall_score, result_triples, individual_scores)
-
+            The evaluation results are returned as a dspy.EvaluationResult object containing the following attributes:
+            
+            - score: A float percentage score (e.g., 67.30) representing overall performance
+            
+            - results: a list of (example, prediction, score) tuples for each example in devset
         """
         metric = metric if metric is not None else self.metric
         devset = devset if devset is not None else self.devset
         num_threads = num_threads if num_threads is not None else self.num_threads
         display_progress = display_progress if display_progress is not None else self.display_progress
         display_table = display_table if display_table is not None else self.display_table
-        return_all_scores = return_all_scores if return_all_scores is not None else self.return_all_scores
-        return_outputs = return_outputs if return_outputs is not None else self.return_outputs
 
         if callback_metadata:
             logger.debug(f"Evaluate is called with callback metadata: {callback_metadata}")
@@ -194,14 +186,11 @@ class Evaluate:
             else:
                 logger.warning("Skipping table display since `pandas` is not installed.")
 
-        if return_all_scores and return_outputs:
-            return round(100 * ncorrect / ntotal, 2), results, [score for *_, score in results]
-        if return_all_scores:
-            return round(100 * ncorrect / ntotal, 2), [score for *_, score in results]
-        if return_outputs:
-            return round(100 * ncorrect / ntotal, 2), results
+        return EvaluationResult(
+            score=round(100 * ncorrect / ntotal, 2),
+            results=results,
+        )
 
-        return round(100 * ncorrect / ntotal, 2)
 
     def _construct_result_table(
         self, results: list[Tuple["dspy.Example", "dspy.Example", Any]], metric_name: str
diff --git a/dspy/teleprompt/bootstrap_finetune.py b/dspy/teleprompt/bootstrap_finetune.py
index 117959e8..5028099f 100644
--- a/dspy/teleprompt/bootstrap_finetune.py
+++ b/dspy/teleprompt/bootstrap_finetune.py
@@ -231,7 +231,6 @@ def bootstrap_trace_data(
         devset=dataset,
         num_threads=num_threads,
         display_progress=True,
-        return_outputs=True,
         provide_traceback=False,  # TODO(check with team)
         max_errors=len(dataset) * 10,  # TODO(check with team)
         failure_score=failure_score,
@@ -290,10 +289,10 @@ def bootstrap_trace_data(
 
                 return failed_pred, trace
 
-    _, outputs = evaluator(wrapped_program, metric=wrapped_metric)
+    results = evaluator(wrapped_program, metric=wrapped_metric).results
 
     data = []
-    for example_ind, (example, prediction, score) in enumerate(outputs):
+    for example_ind, (example, prediction, score) in enumerate(results):
         try:
             prediction, trace = prediction
         except ValueError as ve:
diff --git a/dspy/teleprompt/copro_optimizer.py b/dspy/teleprompt/copro_optimizer.py
index 6a3760d8..65d09dc6 100644
--- a/dspy/teleprompt/copro_optimizer.py
+++ b/dspy/teleprompt/copro_optimizer.py
@@ -225,7 +225,7 @@ class COPRO(Teleprompter):
                         f"At Depth {d+1}/{self.depth}, Evaluating Prompt Candidate #{c_i+1}/{len(candidates_)} for "
                         f"Predictor {p_i+1} of {len(module.predictors())}.",
                     )
-                    score = evaluate(module_clone, devset=trainset, **eval_kwargs)
+                    score = evaluate(module_clone, devset=trainset, **eval_kwargs).score
                     if self.prompt_model:
                         logger.debug(f"prompt_model.inspect_history(n=1) {self.prompt_model.inspect_history(n=1)}")
                     total_calls += 1
diff --git a/dspy/teleprompt/grpo.py b/dspy/teleprompt/grpo.py
index 4a2e7066..7745d264 100644
--- a/dspy/teleprompt/grpo.py
+++ b/dspy/teleprompt/grpo.py
@@ -123,8 +123,6 @@ class GRPO(FinetuneTeleprompter):
                     devset=valset + trainset,
                     num_threads=self.num_threads,
                     display_progress=True,
-                    return_all_scores=True,
-                    return_outputs=False,
                     provide_traceback=False,  # TODO(check with team)
                     max_errors=len(valset)*10,  # TODO(check with team)
                     failure_score=self.failure_score
@@ -134,8 +132,8 @@ class GRPO(FinetuneTeleprompter):
                 else:
                     logger.info(f"Evaluating the student program on the validation set after training step {step_idx + 1}/{self.num_train_steps}")
                 valset_evaluation = valset_evaluator(student, metric=self.metric)
-                trainset_scores = valset_evaluation[1][len(valset):]
-                valset_scores = valset_evaluation[1][:len(valset)]
+                trainset_scores = [r[-1] for r in valset_evaluation.results[len(valset):]]
+                valset_scores = [r[-1] for r in valset_evaluation.results[:len(valset)]]
                 trainset_agg = sum(trainset_scores) / len(trainset_scores)
                 valset_agg = sum(valset_scores) / len(valset_scores)
                 if step_idx == -1:
@@ -151,8 +149,6 @@ class GRPO(FinetuneTeleprompter):
                     devset=valset,
                     num_threads=self.num_threads,
                     display_progress=True,
-                    return_all_scores=True,
-                    return_outputs=False,
                     provide_traceback=False,  # TODO(check with team)
                     max_errors=len(valset)*10,  # TODO(check with team)
                     failure_score=self.failure_score
@@ -163,9 +159,9 @@ class GRPO(FinetuneTeleprompter):
                     logger.info(f"Evaluating the student program on the validation set after training step {step_idx + 1}/{self.num_train_steps}")
                 valset_evaluation = valset_evaluator(student, metric=self.metric)
                 if step_idx == -1:
-                    logger.info(f"Student program validation set score before training loop: {valset_evaluation[0]}")
+                    logger.info(f"Student program validation set score before training loop: {valset_evaluation.score}")
                 else:
-                    logger.info(f"Student program validation set score after training step {step_idx + 1}/{self.num_train_steps}: {valset_evaluation[0]}")
+                    logger.info(f"Student program validation set score after training step {step_idx + 1}/{self.num_train_steps}: {valset_evaluation.score}")
         else:
             # No validation set provided by user
             if self.report_train_scores:
@@ -177,8 +173,6 @@ class GRPO(FinetuneTeleprompter):
                     devset=trainset,
                     num_threads=self.num_threads,
                     display_progress=True,
-                    return_all_scores=True,
-                    return_outputs=False,
                     provide_traceback=False,  # TODO(check with team)
                     max_errors=len(trainset)*10,  # TODO(check with team)
                     failure_score=self.failure_score
@@ -189,9 +183,9 @@ class GRPO(FinetuneTeleprompter):
                     logger.info(f"Evaluating the student program on the validation set after training step {step_idx + 1}/{self.num_train_steps}")
                 valset_evaluation = valset_evaluator(student, metric=self.metric)
                 if step_idx == -1:
-                    logger.info(f"Student program training set score before training loop: {valset_evaluation[0]}")
+                    logger.info(f"Student program training set score before training loop: {valset_evaluation.score}")
                 else:
-                    logger.info(f"Student program training set score after training step {step_idx + 1}/{self.num_train_steps}: {valset_evaluation[0]}")
+                    logger.info(f"Student program training set score after training step {step_idx + 1}/{self.num_train_steps}: {valset_evaluation.score}")
             else:
                 # No valset provided, and not using train as val
                 assert not self.use_train_as_val, "If report_train_scores is False, use_train_as_val must be False."
diff --git a/dspy/teleprompt/infer_rules.py b/dspy/teleprompt/infer_rules.py
index 70611962..21ed154d 100644
--- a/dspy/teleprompt/infer_rules.py
+++ b/dspy/teleprompt/infer_rules.py
@@ -119,9 +119,8 @@ class InferRules(BootstrapFewShot):
             max_errors=effective_max_errors,
             display_table=False,
             display_progress=True,
-            return_all_scores=True,
         )
-        score, _ = evaluate(program, metric=self.metric)
+        score = evaluate(program, metric=self.metric).score
         return score
 
 
diff --git a/dspy/teleprompt/mipro_optimizer_v2.py b/dspy/teleprompt/mipro_optimizer_v2.py
index f4013d48..fe1225f7 100644
--- a/dspy/teleprompt/mipro_optimizer_v2.py
+++ b/dspy/teleprompt/mipro_optimizer_v2.py
@@ -527,9 +527,7 @@ class MIPROv2(Teleprompter):
         )
         logger.info(f"== Trial {1} / {adjusted_num_trials} - Full Evaluation of Default Program ==")
 
-        default_score, _ = eval_candidate_program(
-            len(valset), valset, program, evaluate, self.rng, return_all_scores=True
-        )
+        default_score = eval_candidate_program(len(valset), valset, program, evaluate, self.rng).score
         logger.info(f"Default program score: {default_score}\n")
 
         trial_logs = {}
@@ -579,7 +577,7 @@ class MIPROv2(Teleprompter):
 
             # Evaluate the candidate program (on minibatch if minibatch=True)
             batch_size = minibatch_size if minibatch else len(valset)
-            score = eval_candidate_program(batch_size, valset, candidate_program, evaluate, self.rng)
+            score = eval_candidate_program(batch_size, valset, candidate_program, evaluate, self.rng).score
             total_eval_calls += batch_size
 
             # Update best score and program
@@ -816,7 +814,7 @@ class MIPROv2(Teleprompter):
             param_score_dict, fully_evaled_param_combos
         )
         logger.info(f"Doing full eval on next top averaging program (Avg Score: {mean_score}) from minibatch trials...")
-        full_eval_score = eval_candidate_program(len(valset), valset, highest_mean_program, evaluate, self.rng)
+        full_eval_score = eval_candidate_program(len(valset), valset, highest_mean_program, evaluate, self.rng).score
         score_data.append({"score": full_eval_score, "program": highest_mean_program, "full_eval": True})
 
         # Log full eval as a trial so that optuna can learn from the new results
diff --git a/dspy/teleprompt/random_search.py b/dspy/teleprompt/random_search.py
index ce0272ac..e189b07b 100644
--- a/dspy/teleprompt/random_search.py
+++ b/dspy/teleprompt/random_search.py
@@ -123,7 +123,9 @@ class BootstrapFewShotWithRandomSearch(Teleprompter):
                 display_progress=True,
             )
 
-            score, subscores = evaluate(program, return_all_scores=True)
+            result = evaluate(program)
+
+            score, subscores = result.score, [output[2] for output in result.results]
 
             all_subscores.append(subscores)
 
diff --git a/dspy/teleprompt/teleprompt_optuna.py b/dspy/teleprompt/teleprompt_optuna.py
index 46cc1236..31724de4 100644
--- a/dspy/teleprompt/teleprompt_optuna.py
+++ b/dspy/teleprompt/teleprompt_optuna.py
@@ -48,9 +48,9 @@ class BootstrapFewShotWithOptuna(Teleprompter):
             display_table=False,
             display_progress=True,
         )
-        score = evaluate(program2, return_all_scores=False)
+        result = evaluate(program2)
         trial.set_user_attr("program", program2)
-        return score
+        return result.score
 
     def compile(self, student, *, teacher=None, max_demos, trainset, valset=None):
         import optuna
diff --git a/dspy/teleprompt/utils.py b/dspy/teleprompt/utils.py
index 31f5d19a..4d71259e 100644
--- a/dspy/teleprompt/utils.py
+++ b/dspy/teleprompt/utils.py
@@ -44,31 +44,24 @@ def create_minibatch(trainset, batch_size=50, rng=None):
     return minibatch
 
 
-def eval_candidate_program(batch_size, trainset, candidate_program, evaluate, rng=None, return_all_scores=False):
+def eval_candidate_program(batch_size, trainset, candidate_program, evaluate, rng=None):
     """Evaluate a candidate program on the trainset, using the specified batch size."""
 
     try:
         # Evaluate on the full trainset
         if batch_size >= len(trainset):
-            return evaluate(
-                candidate_program,
-                devset=trainset,
-                return_all_scores=return_all_scores,
-                callback_metadata={"metric_key": "eval_full"},
-            )
+            return evaluate(candidate_program, devset=trainset, callback_metadata={"metric_key": "eval_full"})
         # Or evaluate on a minibatch
         else:
             return evaluate(
                 candidate_program,
                 devset=create_minibatch(trainset, batch_size, rng),
-                return_all_scores=return_all_scores,
-                callback_metadata={"metric_key": "eval_minibatch"},
+                callback_metadata={"metric_key": "eval_minibatch"}
             )
     except Exception:
         logger.error("An exception occurred during evaluation", exc_info=True)
-        if return_all_scores:
-            return 0.0, [0.0] * len(trainset)
-        return 0.0  # TODO: Handle this better, as -ve scores are possible
+        # TODO: Handle this better, as -ve scores are possible
+        return dspy.Prediction(score=0.0, results=[])
 
 
 def eval_candidate_program_with_pruning(


Test patch:
diff --git a/tests/clients/test_cache.py b/tests/clients/test_cache.py
index edff1485..42a12139 100644
--- a/tests/clients/test_cache.py
+++ b/tests/clients/test_cache.py
@@ -280,3 +280,40 @@ async def test_request_cache_decorator_async(cache):
         # Call with different arguments should compute again
         result3 = await test_function(prompt="Different", model="openai/gpt-4o-mini")
         assert result3 == "Response for Different with openai/gpt-4o-mini"
+
+
+def test_cache_consistency_with_lm_call_modifies_the_request(cache):
+    """Test that the cache is consistent with the LM call that modifies the request."""
+    from dspy.clients.cache import request_cache
+
+    # Mock the dspy.cache attribute
+    with patch("dspy.cache", cache):
+        # Define a test function
+        @request_cache()
+        def test_function(**kwargs):
+            del kwargs["field_to_delete"]
+            return kwargs
+
+        # First call should compute the result
+        test_function(field_to_delete="delete", field_to_keep="keep")
+
+        # The cache key should use the original request, not the modified one
+        assert (
+            cache.get(
+                {
+                    "field_to_keep": "keep",
+                    "_fn_identifier": f"{test_function.__module__}.{test_function.__qualname__}",
+                }
+            )
+            is None
+        )
+        assert (
+            cache.get(
+                {
+                    "field_to_keep": "keep",
+                    "field_to_delete": "delete",
+                    "_fn_identifier": f"{test_function.__module__}.{test_function.__qualname__}",
+                }
+            )
+            is not None
+        )
diff --git a/tests/evaluate/test_evaluate.py b/tests/evaluate/test_evaluate.py
index e84f2542..62572685 100644
--- a/tests/evaluate/test_evaluate.py
+++ b/tests/evaluate/test_evaluate.py
@@ -5,7 +5,7 @@ from unittest.mock import patch
 import pytest
 
 import dspy
-from dspy.evaluate.evaluate import Evaluate
+from dspy.evaluate.evaluate import Evaluate, EvaluationResult
 from dspy.evaluate.metrics import answer_exact_match
 from dspy.predict import Predict
 from dspy.utils.callback import BaseCallback
@@ -51,7 +51,7 @@ def test_evaluate_call():
         display_progress=False,
     )
     score = ev(program)
-    assert score == 100.0
+    assert score.score == 100.0
 
 
 @pytest.mark.extra
@@ -91,8 +91,8 @@ def test_multithread_evaluate_call():
         display_progress=False,
         num_threads=2,
     )
-    score = ev(program)
-    assert score == 100.0
+    result = ev(program)
+    assert result.score == 100.0
 
 
 def test_multi_thread_evaluate_call_cancelled(monkeypatch):
@@ -129,11 +129,10 @@ def test_multi_thread_evaluate_call_cancelled(monkeypatch):
             display_progress=False,
             num_threads=2,
         )
-        score = ev(program)
-        assert score == 100.0
+        ev(program)
 
 
-def test_evaluate_call_bad():
+def test_evaluate_call_wrong_answer():
     dspy.settings.configure(lm=DummyLM({"What is 1+1?": {"answer": "0"}, "What is 2+2?": {"answer": "0"}}))
     devset = [new_example("What is 1+1?", "2"), new_example("What is 2+2?", "4")]
     program = Predict("question -> answer")
@@ -142,8 +141,8 @@ def test_evaluate_call_bad():
         metric=answer_exact_match,
         display_progress=False,
     )
-    score = ev(program)
-    assert score == 0.0
+    result = ev(program)
+    assert result.score == 0.0
 
 
 @pytest.mark.extra
@@ -247,9 +246,13 @@ def test_evaluate_callback():
         metric=answer_exact_match,
         display_progress=False,
     )
-    score = ev(program)
-    assert score == 100.0
+    result = ev(program)
+    assert result.score == 100.0
     assert callback.start_call_inputs["program"] == program
     assert callback.start_call_count == 1
-    assert callback.end_call_outputs == 100.0
+    assert callback.end_call_outputs.score == 100.0
     assert callback.end_call_count == 1
+
+def test_evaluation_result_repr():
+    result = EvaluationResult(score=100.0, results=[(new_example("What is 1+1?", "2"), {"answer": "2"}, 100.0)])
+    assert repr(result) == "EvaluationResult(score=100.0, results=<list of 1 results>)"
diff --git a/tests/teleprompt/test_utils.py b/tests/teleprompt/test_utils.py
index 5b3a5909..443dc2bd 100644
--- a/tests/teleprompt/test_utils.py
+++ b/tests/teleprompt/test_utils.py
@@ -1,7 +1,5 @@
 from unittest.mock import Mock
 
-import pytest
-
 import dspy
 from dspy.teleprompt.utils import eval_candidate_program
 
@@ -43,19 +41,12 @@ def test_eval_candidate_program_minibatch():
     assert called_kwargs["callback_metadata"] == {"metric_key": "eval_minibatch"}
     assert result == 0
 
-
-@pytest.mark.parametrize("return_all_scores", [True, False])
-def test_eval_candidate_program_failure(return_all_scores):
+def test_eval_candidate_program_failure():
     trainset = [1, 2, 3, 4, 5]
     candidate_program = DummyModule()
     evaluate = Mock(side_effect=ValueError("Error"))
     batch_size = 3
 
-    result = eval_candidate_program(
-        batch_size, trainset, candidate_program, evaluate, return_all_scores=return_all_scores
-    )
+    result = eval_candidate_program(batch_size, trainset, candidate_program, evaluate)
 
-    if return_all_scores:
-        assert result == (0.0, [0.0] * len(trainset))
-    else:
-        assert result == 0.0
+    assert result.score == 0.0