Repository: ErikBjare/gptme
File: gptme/eval/dspy/prompt_optimizer.py (Python)

Code before:
from .signatures import GptmeTaskSignature, PromptImprovementSignature

logger = logging.getLogger(__name__)


class ModelNameMapper:
    """Handles mapping between gptme and DSPy model names."""

    @staticmethod
    def to_dspy_format(gptme_model: str) -> str:
        """Convert gptme model name to DSPy/litellm format."""
        if gptme_model.startswith("anthropic/"):
            return gptme_model.replace("anthropic/", "")
        elif gptme_model.startswith("openai/"):
            return gptme_model.replace("openai/", "")
        else:
            return gptme_model

    @staticmethod
    def get_reflection_model(base_model: str) -> str:
        """Get a more powerful model for reflection tasks."""
        if base_model.startswith("anthropic/"):
            if "haiku" in base_model.lower():
                return "anthropic/claude-sonnet-4-5"
            else:
                return "anthropic/claude-sonnet-4-5"
        elif base_model.startswith("openai/"):
            if "gpt-3.5" in base_model.lower() or "gpt-4o-mini" in base_model.lower():
                return "openai/gpt-4o"
            else:
                return "openai/gpt-4o"
        else:
            return ModelNameMapper.to_dspy_format(base_model)


class PromptDataset:
    """Dataset wrapper for gptme evaluation tasks."""

    def __init__(self, eval_specs: list[EvalSpec], limit: int | None = None):
        if not eval_specs:
            eval_specs = tests
        self.eval_specs = eval_specs[:limit] if limit else eval_specs

    def __len__(self) -> int:
        return len(self.eval_specs)

    def __iter__(self):
        for spec in self.eval_specs:
            yield self._spec_to_example(spec)

    def _spec_to_example(self, spec: EvalSpec) -> dspy.Example:

Review comment (suggestion):
```suggestion
            return "claude-sonnet-4-5"
```

Code after:
from .signatures import GptmeTaskSignature, PromptImprovementSignature

logger = logging.getLogger(__name__)


class ModelNameMapper:
    """Handles mapping between gptme and DSPy model names."""

    @staticmethod
    def to_dspy_format(gptme_model: str) -> str:
        """Convert gptme model name to DSPy/litellm format."""
        if gptme_model.startswith("anthropic/"):
            return gptme_model.replace("anthropic/", "")
        elif gptme_model.startswith("openai/"):
            return gptme_model.replace("openai/", "")
        else:
            return gptme_model

    @staticmethod
    def get_reflection_model(base_model: str) -> str:
        """Get a more powerful model for reflection tasks."""
        if base_model.startswith("anthropic/"):
            return "claude-sonnet-4-5"
        elif base_model.startswith("openai/"):
            return "gpt-5"
        else:
            return ModelNameMapper.to_dspy_format(base_model)


class PromptDataset:
    """Dataset wrapper for gptme evaluation tasks."""

    def __init__(self, eval_specs: list[EvalSpec], limit: int | None = None):
        if not eval_specs:
            eval_specs = tests
        self.eval_specs = eval_specs[:limit] if limit else eval_specs

    def __len__(self) -> int:
        return len(self.eval_specs)

    def __iter__(self):
        for spec in self.eval_specs:
            yield self._spec_to_example(spec)

    def _spec_to_example(self, spec: EvalSpec) -> dspy.Example:
        """Convert an eval spec to a DSPy example, preserving the original spec."""
        return dspy.Example(
            task_description=spec.get("prompt", ""),
            context=self._build_context(spec),
            eval_spec=spec,  # Keep original spec for actual evaluation
            name=spec.get("name", "unknown"),