Repository: microsoft/promptbase
File: guidance_programs/fewshot_cot_as_conversation_ensemble.py (Python)

Code before:
    with assistant():
        lm += gen(name=COT_KEY)

    with user():
        lm += f"**Final Answer**"

    with assistant():
        lm += select([str(i) for i in range(len(choices))], name=ANSWER_KEY)

    return lm


def guidance_generation(
    lm: guidance.models.Chat,
    input: Dict[str, Any],
    common: list[dict[str, Any]] | None = None,
) -> Dict[str, Any]:
    _logger.debug("Starting guidance_generation")
    assert common is None, "Unexpected common data"

    votes = [0, 0, 0, 0]
    cots = []
    generator = plain_hunt_generator(list(range(len(input["choices"]))))
    for i in range(NUM_PERMUTATIONS):
        current_permutation = next(generator)
        result = lm + few_shot_cot_multiple_choice(
            question=input["question"],
            choices=input["choices"],
            fewshot_examples=input["fewshot_examples"],
            permutation=current_permutation,
        )
        _logger.debug(f"Result: {result}")
        cots.append(result[COT_KEY])
        selected = int(result[ANSWER_KEY])
        actual = current_permutation[selected]
        votes[actual] += 1

    _logger.debug(f"Votes: {votes}")
    # Check the votes
    max_idx = -1
    curr_max = 0
    for i in range(len(votes)):
        if votes[i] > curr_max:
            curr_max = votes[i]
            max_idx = i

    final_result = dict(fewshot_choice=max_idx, fewshot_cot=cots)
    _logger.debug(f"final_result: {final_result}")
    return final_result


Review comment (suggestion):
As we discussed yesterday @Harsha-Nori , the `guidance` program is just Python, so we can sneak things like this in :-)

Code after:
    with assistant():
        lm += gen(name=COT_KEY)

    with user():
        lm += f"**Final Answer**"

    with assistant():
        lm += select([str(i) for i in range(len(choices))], name=ANSWER_KEY)

    return lm


def guidance_generation(
    lm: guidance.models.Chat,
    input: Dict[str, Any],
    common: list[dict[str, Any]] | None = None,
) -> Dict[str, Any]:
    _logger.debug("Starting guidance_generation")
    assert common is None, "Unexpected common data"

    num_choices = len(input["choices"])

    votes = [0 for _ in range(num_choices)]
    cots = []
    generator = plain_hunt_generator(list(range(num_choices)))
    for i in range(NUM_PERMUTATIONS):
        current_permutation = next(generator)
        result = lm + few_shot_cot_multiple_choice(
            question=input["question"],
            choices=input["choices"],
            fewshot_examples=input["fewshot_examples"],
            permutation=current_permutation,
        )
        _logger.debug(f"Result: {result}")
        cots.append(result[COT_KEY])
        selected = int(result[ANSWER_KEY])
        actual = current_permutation[selected]
        votes[actual] += 1

    _logger.debug(f"Votes: {votes}")
    # Check the votes
    max_idx = -1
    curr_max = 0
    for i in range(len(votes)):
        if votes[i] > curr_max:
            curr_max = votes[i]
            max_idx = i

    final_result = dict(fewshot_choice=max_idx, fewshot_cot=cots)
    _logger.debug(f"final_result: {final_result}")
    return final_result