Repository: oumi-ai/oumi
File: src/oumi/core/configs/params/judge_params.py (Python)

Code before:
    expected output fields. Used to provide examples of how the judge should respond.

    Example:
        [
            {
                "question": "What is 2+2?",                      # placeholder value
                "answer": "4",                                   # placeholder value
                "judgment": "Correct",                           # output field value
                "explanation": "It is mathematically correct."   # output field value
            },
            {
                "question": "What is the capital of Mars?",      # placeholder value
                "answer": "New York",                            # placeholder value
                "judgment": "Incorrect",                         # output field value
                "explanation": "Mars does not have capitals."    # output field value
            }
        ]
    """

    def __post_init__(self):
        """Validate the parameters after initialization."""
        self._validate_params()
        self._replace_template_variables()

    def _replace_template_variables(self):
        """Apply template variables to prompt_template and system_instruction."""
        VARIABLES_REGEX = r"\{(\w+)\}"

        if not self.template_variables:
            return

        # Find all variables in prompt_template and system_instruction
        all_variable_names = set(re.findall(VARIABLES_REGEX, self.prompt_template))
        all_variable_names.update(
            re.findall(VARIABLES_REGEX, self.system_instruction or "")
        )

        unused_variables = set(self.template_variables.keys()) - all_variable_names
        if unused_variables:
            raise ValueError(
                "The following template variables are not used in the prompt_template "
                f"or system_instruction: {sorted(unused_variables)}"
            )

        # Replace the template variables with their values
        for variable_name, variable_value in self.template_variables.items():
            self.prompt_template = self.prompt_template.replace(
                f"{{{variable_name}}}", variable_value
            )
            if self.system_instruction:
                self.system_instruction = self.system_instruction.replace(

Review comment (suggestion):
Thoughts on replacing the traditional format string {word} with something more unique like <<word>>?

Also, could we segment this logic out and add support for . characters? For example:

<<attribute_id.value.description>>

I'll ultimately need this logic for synthesis, so it'd be good if we standardize. I'm not strongly opinioned on {} vs <<>>, but I'd like . support preferably.

Code after:
    expected output fields. Used to provide examples of how the judge should respond.

    Example:
        [
            {
                "question": "What is 2+2?",                      # placeholder value
                "answer": "4",                                   # placeholder value
                "judgment": "Correct",                           # output field value
                "explanation": "It is mathematically correct."   # output field value
            },
            {
                "question": "What is the capital of Mars?",      # placeholder value
                "answer": "New York",                            # placeholder value
                "judgment": "Incorrect",                         # output field value
                "explanation": "Mars does not have capitals."    # output field value
            }
        ]
    """

    def __post_init__(self):
        """Validate the parameters after initialization."""
        self._validate_params()

    def _validate_params(self):
        """Validate the parameters for consistency and completeness.

        Raises:
            ValueError: If parameters are invalid
        """
        # Validate prompt template is not empty
        if not self.prompt_template.strip():
            raise ValueError("prompt_template cannot be empty")

        # Validate judgment scores for ENUM judgment type
        if self.judgment_type == JudgeOutputType.ENUM and not self.judgment_scores:
            raise ValueError("judgment_scores must be provided for ENUM judgment_type")

        # Validate judgment scores are numeric if provided
        if self.judgment_scores:
            if not all(
                isinstance(score, (int, float))
                for score in self.judgment_scores.values()
            ):
                raise ValueError("All judgment_scores values must be numeric")
            if not self.judgment_scores:
                raise ValueError("judgment_scores cannot be empty when provided")

    def replace_template_variables(self):
        """Apply template variables to prompt_template and system_instruction."""
        if not self.template_variables:
            return