reval_ana3.py

import os
import json
from tqdm import tqdm
from textgames import GAME_NAMES, game_filename, _game_class_from_name
from pathlib import Path

GAME_NAME = GAME_NAMES[5]
PROBLEMSET_DIR = Path(os.getenv("TG_PROBLEMSET_DIR", "problemsets"))
MODEL_OUTPUT_DIR = Path(os.getenv("TG_MODEL_OUTPUT_DIR", "model_outputs"))
OUTPUT_FILENAMES = [
    # "results_gemma-2-9b-it.1s.jsonl",
    # "results_gemma-2-9b-it.zs.jsonl",
    # "results_gemma-2-27b-it.1s.jsonl",
    # "results_gemma-2-27b-it.zs.jsonl",
    #
    # "results_llama-3.1-8b-instruct.1s.jsonl",
    # "results_llama-3.1-8b-instruct.zs.jsonl",
    # "results_llama-3.1-70b-instruct.1s.jsonl",
    # "results_llama-3.1-70b-instruct.zs.jsonl",
    # "results_llama-3.3-70b-instruct.1s.jsonl",
    # "results_llama-3.3-70b-instruct.zs.jsonl",
    #
    # "results_qwen2-5-7b-instruct.1s.jsonl",
    # "results_qwen2-5-7b-instruct.zs.jsonl",
    # "results_qwen2-5-14b-instruct.1s.jsonl",
    # "results_qwen2-5-14b-instruct.zs.jsonl",
    # "results_qwen2-5-32b-instruct.1s.jsonl",
    # "results_qwen2-5-32b-instruct.zs.jsonl",
    # "results_qwen2-5-72b-instruct.1s.jsonl",
    # "results_qwen2-5-72b-instruct.zs.jsonl",
    #
    # "results_deepseek-r1-distill-14b.1s.jsonl",
    # "results_deepseek-r1-distill-14b.zs.jsonl",
    # "results_deepseek-r1-distill-14b.rerun.1s.jsonl",
    #
    # "results_chatgpt-4o-mini.zs.jsonl",
    # "results_chatgpt-o3-mini.zs.jsonl",
    #
    # "results_qwen2-5-7b-instruct_sp.1s.jsonl",
    # "results_qwen2-5-7b-instruct_sp.zs.jsonl",

    # "results_deepseek-r1-distill-8b.1s.jsonl",
    "results_deepseek-r1-distill-8b.zs.jsonl",
]

# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# !!! Must run reval_bracket_rerun.py first !!!
# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!


def revalidate_anagram_3(fp, reval_dir="revalidate_anagram_3", source_dir="prior_revalidate"):
    os.makedirs(MODEL_OUTPUT_DIR/reval_dir, exist_ok=True)
    count_pos, count_neg = 0, 0
    with (open(MODEL_OUTPUT_DIR/source_dir/fp, "r", encoding="utf8") as i,
          open(MODEL_OUTPUT_DIR/reval_dir/fp, "w", encoding="utf8") as o,
          tqdm(total=1000, desc=fp) as pbar,
          ):
        for line in i:
            res = json.loads(line)
            if (res['game'] == f"{game_filename(GAME_NAME)}_3"):
                if (res['turn'] == 1):
                    cur_sid = res["session"]
                    prompt = sid_prompt_dict[cur_sid]
                    cur_game = game_cls()
                    cur_game.load_game(prompt)
                    pbar.update(1)
                elif solved == True:
                    continue
                else:
                    assert cur_sid == res["session"]
                solved, _ = cur_game.validate(res["response"])
                if solved and not res["solved"]:
                    count_pos += 1
                elif not solved and res["solved"]:
                    count_neg += 1
                res["solved"] = solved
            o.write(json.dumps(res))
            o.write("\n")
    return count_pos, count_neg


if __name__ == "__main__":
    game_cls = _game_class_from_name(GAME_NAME)
    with open(f"{PROBLEMSET_DIR}/{game_filename(GAME_NAME)}_3.json", "r", encoding="utf8") as f:
        sid_prompt_dict = json.load(f)
    for fp in OUTPUT_FILENAMES:
        print(revalidate_anagram_3(fp))