Repository: oumi-ai/oumi
File: src/oumi/judges/judge_court.py (Python)

Code before:
    """Returns a JudgeConfig for the Oumi v1 XML DeepSeek R1 judge.

    This function creates and returns a JudgeConfig object for the Oumi V1 Judge, which
    uses DeepSeek R1 as a judge, with inputs and outputs in XML format.

    Returns:
        JudgeConfig: A configuration object for the Oumi v1 XML DeepSeek R1 judge.

    Note:
        This judge uses the Together Cloud API, so the TOGETHER_API_KEY environment
        variable must be set with a valid API key.
    """
    judges_directory = get_oumi_root_directory() / "judges" / "oumi_v1"

    attribute_names = ["helpful", "honest", "safe"]

    attributes = {
        attribute: JudgeAttribute[Union[OumiJudgeInput, OumiJudgeOutput]].load(
            str(judges_directory / f"{attribute}.json")
        )
        for attribute in attribute_names
    }

    config = JudgeConfig(
        attributes=attributes,
        model=ModelParams(model_name="deepseek-ai/DeepSeek-R1"),
        engine=InferenceEngineType.REMOTE,
        generation=GenerationParams(
            max_new_tokens=1024,
            temperature=0.0,
        ),
        remote_params=RemoteParams(
            api_url="https://api.together.xyz/v1/chat/completions",
            api_key_env_varname="TOGETHER_API_KEY",
            max_retries=3,
            # Experience number. Based on curl test, it takes about 48 seconds to 
            # complete a LLM as judge task
            connection_timeout=60, 
            # Note: Due to high demand, DeepSeek R1 has a model specific rate limit of \
            # 6 RPM for Free Users, and 12 RPM for Build Tier 1 users.
            # https://docs.together.ai/docs/rate-limits#rate-limit-tiers
            politeness_policy=11.0
        ),
    )
    return config

@register_judge("oumi/v1_xml_unit_test")
def unit_test_judge():
    """Tiny judge for unit testing.

    Do not use this judge for anything serious as it returns random results.

Review comment (suggestion):
@ctseng777 you can use the TogetherInferenceEngine directly, using `InferenceEngineType. TOGETHER`. This way you don't have to manually set api_url and api_key_env_varname !

Same with SambaNova & DeepSeek

Code after:
        JudgeConfig: A configuration object for the Oumi v1 XML DeepSeek R1 judge.

    Note:
        This judge uses the Together Cloud API, so the TOGETHER_API_KEY environment
        variable must be set with a valid API key.
    """
    judges_directory = get_oumi_root_directory() / "judges" / "oumi_v1"

    attribute_names = ["helpful", "honest", "safe"]

    attributes = {
        attribute: JudgeAttribute[Union[OumiJudgeInput, OumiJudgeOutput]].load(
            str(judges_directory / f"{attribute}.json")
        )
        for attribute in attribute_names
    }

    config = JudgeConfig(
        attributes=attributes,
        model=ModelParams(model_name="deepseek-ai/DeepSeek-R1"),
        engine=InferenceEngineType.TOGETHER,
        generation=GenerationParams(
            max_new_tokens=1024,
            temperature=0.0,
        ),
        remote_params=RemoteParams(
            max_retries=3,
            # Experience number. Based on curl test, it takes about 48 seconds to
            # complete a LLM as judge task
            connection_timeout=60,
            # Note: Due to high demand, DeepSeek R1 has a model specific rate limit of \
            # 6 RPM for Free Users, and 12 RPM for Build Tier 1 users.
            # https://docs.together.ai/docs/rate-limits#rate-limit-tiers
            politeness_policy=11.0,
        ),
    )
    return config


@register_judge("oumi/v1_xml_unit_test")
def unit_test_judge():
    """Tiny judge for unit testing.

    Do not use this judge for anything serious as it returns random results.
    """
    attribute_path = (
        get_oumi_root_directory() / "judges" / "test_judge" / "helpful.json"
    )

    attribute = JudgeAttribute[Union[OumiJudgeInput, OumiJudgeOutput]].load(
        str(attribute_path)