Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+textgames-scrabble-black2-ss.png filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,10 +1,9 @@
 */*.DS_Store
 .DS_Store
 
-ssl/
-problemsets_*
-
+agents/*.sh
 user_outputs/
+model_outputs/__runs__
 
 .idea/
 

agents/Gemma-2-9b-it.ipynb

@@ -0,0 +1 @@
+{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[]},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"code","execution_count":1,"metadata":{"id":"Rli_enT6lBDT","executionInfo":{"status":"ok","timestamp":1737395007014,"user_tz":-540,"elapsed":5212,"user":{"displayName":"Frederikus Hudi","userId":"06160664103998835801"}}},"outputs":[],"source":["##%%\n","import os\n","import pickle\n","import json\n","# import random\n","# import torch\n","# import numpy as np\n","# import argparse\n","# import cohere\n","# from openai import OpenAI\n"]},{"cell_type":"code","source":["##%%\n","# import hashlib\n","from tqdm import tqdm\n","from itertools import product\n","# from collections import Counter\n","\n","# from transformers import LlamaForCausalLM, AutoTokenizer, AutoModelForCausalLM, AutoModelForSeq2SeqLM\n","from transformers import AutoTokenizer, AutoModelForCausalLM\n","from textgames import GAME_NAMES, LEVEL_IDS, game_filename, _game_class_from_name\n"],"metadata":{"id":"dp1F32B8oSfD","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1737395010583,"user_tz":-540,"elapsed":3547,"user":{"displayName":"Frederikus Hudi","userId":"06160664103998835801"}},"outputId":"e9adeb5f-70eb-4ca9-dcbb-428e4b28ab41"},"execution_count":2,"outputs":[{"output_type":"stream","name":"stderr","text":["/home/is/frederikus-h/miniconda3/envs/textgame/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n","  from .autonotebook import tqdm as notebook_tqdm\n"]}]},{"cell_type":"code","source":["os.environ.setdefault(\"TEXTGAMES_OUTPUT_DIR\", \"user_outputs\")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"2wEu1V1wvxn0","executionInfo":{"status":"ok","timestamp":1737395010664,"user_tz":-540,"elapsed":67,"user":{"displayName":"Frederikus Hudi","userId":"06160664103998835801"}},"outputId":"cdcad20f-e357-4009-9f4f-0d4495ebd894"},"execution_count":3,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'user_outputs'"]},"metadata":{},"execution_count":3}]},{"cell_type":"code","source":["##%%\n","gen_model_checkpoint = \"google/gemma-2-9b-it\"\n","quantize = True"],"metadata":{"id":"jZF8bkUcojTX","executionInfo":{"status":"ok","timestamp":1737395010678,"user_tz":-540,"elapsed":13,"user":{"displayName":"Frederikus Hudi","userId":"06160664103998835801"}}},"execution_count":4,"outputs":[]},{"cell_type":"code","source":["kwargs = {\n","    \"device_map\": \"auto\",\n","} if quantize else {}"],"metadata":{"id":"VAF5sR9arYzS","executionInfo":{"status":"ok","timestamp":1737395010683,"user_tz":-540,"elapsed":2,"user":{"displayName":"Frederikus Hudi","userId":"06160664103998835801"}}},"execution_count":5,"outputs":[]},{"cell_type":"code","source":["##%%\n","gen_model = AutoModelForCausalLM.from_pretrained(gen_model_checkpoint, **kwargs)\n","tokenizer = AutoTokenizer.from_pretrained(gen_model_checkpoint, **kwargs)"],"metadata":{"id":"tzqldl8ooRVL","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1737395038547,"user_tz":-540,"elapsed":27859,"user":{"displayName":"Frederikus Hudi","userId":"06160664103998835801"}},"outputId":"902b638c-e6ce-4f8a-bba2-e9f7241c9a27"},"execution_count":6,"outputs":[{"output_type":"stream","name":"stderr","text":["Loading checkpoint shards: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4/4 [00:24<00:00,  6.19s/it]\n"]}]},{"cell_type":"code","source":["gen_model.device"],"metadata":{"id":"FeBUXdkWsWrL","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1737395038552,"user_tz":-540,"elapsed":3,"user":{"displayName":"Frederikus Hudi","userId":"06160664103998835801"}},"outputId":"6437d1b7-02f8-47f5-d519-e979cefde795"},"execution_count":7,"outputs":[{"output_type":"execute_result","data":{"text/plain":["device(type='cuda', index=0)"]},"metadata":{},"execution_count":7}]},{"cell_type":"code","source":["def get_gemma_response(text):\n","    # global gen_model, tokenizer\n","    messages = [\n","        {\"role\": \"user\", \"content\": text},\n","    ]\n","\n","    input_ids = tokenizer.apply_chat_template(\n","        messages,\n","        add_generation_prompt=True,\n","        return_tensors=\"pt\"\n","    ).to(gen_model.device)\n","\n","    terminators = [\n","        tokenizer.eos_token_id,\n","        tokenizer.convert_tokens_to_ids(\"<|eot_id|>\")\n","    ]\n","\n","    outputs = gen_model.generate(\n","        input_ids,\n","        max_new_tokens=100,\n","        eos_token_id=terminators,\n","        do_sample=True,\n","        temperature=.001,\n","        top_p=1,\n","    )\n","\n","    response = outputs[0][input_ids.shape[-1]:]\n","    return tokenizer.decode(response, skip_special_tokens=True)"],"metadata":{"id":"R5D4K-P2sPaj","executionInfo":{"status":"ok","timestamp":1737395038554,"user_tz":-540,"elapsed":1,"user":{"displayName":"Frederikus Hudi","userId":"06160664103998835801"}}},"execution_count":8,"outputs":[]},{"cell_type":"markdown","source":["---\n","Example Call"],"metadata":{"id":"s5FEwOOvxf4h"}},{"cell_type":"code","source":["# @title\n","text = \\\n","\"\"\"\n","Given a set of rules to calculate point, sort the set of words in decreasing order.\n","When there 2 or more words with same point, sort lexicographically.\n","\n","Rules:\n","- every pair of consecutive consonant gets 5 points\n","- every pair of consecutive vowel gets 3 points\n","- add 1 point if there exists exactly 1 'g' in the word\n","- word less than 5 characters gets 10 points\n","- word starts with gen gets 100 points\n","- word ends with ta gets -1000 point\n","\n","Words:\n","- genta\n","- winata\n","- hudi\n","- alham\n","- aji\n","- ruochen\n","\n","Print only the answer.\n","\"\"\"\n","\n","print(text)"],"metadata":{"id":"T_tk4hTGsxsR","colab":{"base_uri":"https://localhost:8080/"},"cellView":"form","executionInfo":{"status":"ok","timestamp":1737392776367,"user_tz":-540,"elapsed":27,"user":{"displayName":"Frederikus Hudi","userId":"06160664103998835801"}},"outputId":"d5ea884f-d0fa-4134-ecd9-690eab51c976"},"execution_count":14,"outputs":[{"output_type":"stream","name":"stdout","text":["\n","Given a set of rules to calculate point, sort the set of words in decreasing order.\n","When there 2 or more words with same point, sort lexicographically.\n","\n","Rules:\n","- every pair of consecutive consonant gets 5 points\n","- every pair of consecutive vowel gets 3 points\n","- add 1 point if there exists exactly 1 'g' in the word\n","- word less than 5 characters gets 10 points\n","- word starts with gen gets 100 points\n","- word ends with ta gets -1000 point\n","\n","Words:\n","- genta\n","- winata\n","- hudi\n","- alham\n","- aji\n","- ruochen\n","\n","Print only the answer.\n","\n"]}]},{"cell_type":"code","source":["# Gold Answer:\n","# - aji      10\n","# - hudi     10\n","# - ruochen   5  3\n","# - alham     5\n","# - genta     5  1  100 -1000\n","# - winata  -1000"],"metadata":{"id":"G-5yS4S-rdsN"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["print(get_gemma_response(text))"],"metadata":{"id":"05OI36v6vGoY","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1737392724119,"user_tz":-540,"elapsed":6741,"user":{"displayName":"Frederikus Hudi","userId":"06160664103998835801"}},"outputId":"fe5d6ed2-d063-4f1c-b2e1-b3af8dbc456e"},"execution_count":9,"outputs":[{"output_type":"stream","name":"stdout","text":["genta\n","winata\n","ruochen\n","hudi\n","alham\n","aji \n","\n"]}]},{"cell_type":"markdown","source":["---\n","Automate run all sessions"],"metadata":{"id":"cxJ4WqHpxi75"}},{"cell_type":"code","source":["for game_name, difficulty_level in product([GAME_NAMES[4], *GAME_NAMES[:4], *GAME_NAMES[5:]], LEVEL_IDS[:3]):\n","    game_cls = _game_class_from_name(game_name)\n","    with open(f\"problemsets/{game_filename(game_name)}_{difficulty_level}.json\", \"r\", encoding=\"utf8\") as f:\n","        sid_prompt_dict = json.load(f)\n","\n","    correct_cnt = 0\n","    for sid, prompt in tqdm(list(sid_prompt_dict.items()), desc=f\"{game_filename(game_name)}_-_{difficulty_level}\"):\n","        cur_game = game_cls()\n","        cur_game.load_game(prompt)\n","        response = get_gemma_response(cur_game.get_prompt()).strip()\n","        solved, val_msg = cur_game.validate(response)\n","        with open(f\"model_outputs/results_gemma_2_9B_it.pkl\", \"ab\") as o:\n","            pickle.dump((f\"{game_filename(game_name)}_{difficulty_level}\", sid, response, (solved, val_msg)), o)\n","        if solved:\n","            correct_cnt += 1\n","\n","    print(f\"{game_name}_-_{difficulty_level}\")\n","    print(f\"  Acc.: {correct_cnt / len(sid_prompt_dict):.2%}\")"],"metadata":{"id":"hCTXYpXa1UQ6"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":[],"metadata":{"id":"GC-zkVI52IJX"},"execution_count":null,"outputs":[]}]}

agents/__init__.py

@@ -0,0 +1,5 @@
+# Define the __all__ variable
+__all__ = ["run_with_agent"]
+
+# Import the submodules
+from .runner import run_with_agent

agents/_reference.py

@@ -0,0 +1,216 @@
+import os
+import torch
+import random
+import numpy as np
+import argparse
+import json
+import cohere
+from openai import OpenAI
+
+from tqdm import tqdm
+
+from collections import Counter
+
+from transformers import LlamaForCausalLM, AutoTokenizer, AutoModelForCausalLM, AutoModelForSeq2SeqLM
+import hashlib
+
+
+OPENAI_TOKEN = ""
+COHERE_TOKEN = ""
+HF_TOKEN = ""
+
+
+def argmax(array):
+    """argmax with deterministic pseudorandom tie breaking."""
+    max_indices = np.arange(len(array))[array == np.max(array)]
+    idx = int(hashlib.sha256(np.asarray(array).tobytes()).hexdigest(), 16) % len(max_indices)
+    return max_indices[idx]
+
+
+def logsumexp(x):
+    c = x.max()
+    return c + np.log(np.sum(np.exp(x - c)))
+
+
+def normalize(x):
+    x = np.array(x)
+    return np.exp(x - logsumexp(x))
+
+
+def set_seed(seed):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+
+
+def get_commandr_chat_response(gen_model, gen_model_checkpoint, text, seed):
+    response = gen_model.chat(
+        model="command-r",
+        message=text,
+        temperature=0,
+        max_tokens=64,
+        seed=seed,
+        p=1
+    )
+    return response.text
+
+
+def get_mt0_response(gen_model, tokenizer, gen_model_checkpoint, text, seed):
+    input_ids = tokenizer.encode(text, return_tensors="pt").to(gen_model.device)
+
+    outputs = gen_model.generate(
+        input_ids,
+        max_new_tokens=10,
+        do_sample=True,
+        temperature=0.2,
+        top_p=1
+    )
+
+    response = outputs[0]
+    return tokenizer.decode(response, skip_special_tokens=True)
+
+
+def get_gemma_response(gen_model, tokenizer, gen_model_checkpoint, text, seed):
+    messages = [
+        {"role": "user", "content": text},
+    ]
+
+    input_ids = tokenizer.apply_chat_template(
+        messages,
+        add_generation_prompt=True,
+        return_tensors="pt"
+    ).to(gen_model.device)
+
+    terminators = [
+        tokenizer.eos_token_id,
+        tokenizer.convert_tokens_to_ids("<|eot_id|>")
+    ]
+
+    outputs = gen_model.generate(
+        input_ids,
+        max_new_tokens=10,
+        eos_token_id=terminators,
+        do_sample=True,
+        temperature=0.2,
+        top_p=1
+    )
+
+    response = outputs[0][input_ids.shape[-1]:]
+    return tokenizer.decode(response, skip_special_tokens=True)
+
+
+def get_mistral_instruct_chat_response(gen_model, tokenizer, gen_model_checkpoint, text, seed):
+    messages = [
+        {"role": "user", "content": text},
+    ]
+
+    input_ids = tokenizer.apply_chat_template(
+        messages,
+        add_generation_prompt=True,
+        return_tensors="pt"
+    ).to(gen_model.device)
+
+    terminators = [
+        tokenizer.eos_token_id,
+        tokenizer.convert_tokens_to_ids("<|eot_id|>")
+    ]
+
+    outputs = gen_model.generate(
+        input_ids,
+        max_new_tokens=10,
+        eos_token_id=terminators,
+        do_sample=True,
+        temperature=0.2,
+        top_p=1
+    )
+
+    response = outputs[0][input_ids.shape[-1]:]
+    return tokenizer.decode(response, skip_special_tokens=True)
+
+
+def get_llama3_instruct_chat_response(gen_model, tokenizer, gen_model_checkpoint, text, seed):
+    messages = [
+        {"role": "user", "content": text},
+    ]
+
+    input_ids = tokenizer.apply_chat_template(
+        messages,
+        add_generation_prompt=True,
+        return_tensors="pt"
+    ).to(gen_model.device)
+
+    terminators = [
+        tokenizer.eos_token_id,
+        tokenizer.convert_tokens_to_ids("<|eot_id|>")
+    ]
+
+    outputs = gen_model.generate(
+        input_ids,
+        max_new_tokens=10,
+        eos_token_id=terminators,
+        do_sample=True,
+        temperature=0.2,
+        top_p=1
+    )
+
+    response = outputs[0][input_ids.shape[-1]:]
+    return tokenizer.decode(response, skip_special_tokens=True)
+
+
+def get_openai_chat_response(gen_model, gen_model_checkpoint, text, seed):
+    messages = [
+        {
+            "role": "user",
+            "content": text
+        }
+    ]
+    response = gen_model.chat.completions.create(
+        model=gen_model_checkpoint,
+        messages=messages,
+        temperature=0,
+        max_tokens=64,
+        top_p=1,
+        seed=seed
+    )
+    return response.choices[0].message.content
+
+
+def load_model(gen_model_checkpoint, load_in_8bit=False):
+    gen_model = None
+    tokenizer = None
+
+    if "mistralai/Mistral-7B-Instruct-v0.3" in gen_model_checkpoint or "meta-llama/Meta-Llama-3-8B-Instruct" in gen_model_checkpoint or "google/gemma-1.1-7b-it" in gen_model_checkpoint:
+        if load_in_8bit:
+            gen_model = AutoModelForCausalLM.from_pretrained(gen_model_checkpoint, token=HF_TOKEN, device_map="auto",
+                                                             load_in_8bit=True)
+            tokenizer = AutoTokenizer.from_pretrained(gen_model_checkpoint, token=HF_TOKEN, device_map="auto",
+                                                      load_in_8bit=True)
+        else:
+            gen_model = AutoModelForCausalLM.from_pretrained(gen_model_checkpoint, token=HF_TOKEN)
+            tokenizer = AutoTokenizer.from_pretrained(gen_model_checkpoint, token=HF_TOKEN)
+    elif "CohereForAI/aya-101" in gen_model_checkpoint or "bigscience/mt0" in gen_model_checkpoint:
+        if load_in_8bit:
+            gen_model = AutoModelForSeq2SeqLM.from_pretrained(gen_model_checkpoint, token=HF_TOKEN, device_map="auto",
+                                                              load_in_8bit=True)
+            tokenizer = AutoTokenizer.from_pretrained(gen_model_checkpoint, token=HF_TOKEN, device_map="auto",
+                                                      load_in_8bit=True)
+        else:
+            gen_model = AutoModelForSeq2SeqLM.from_pretrained(gen_model_checkpoint, token=HF_TOKEN)
+            tokenizer = AutoTokenizer.from_pretrained(gen_model_checkpoint, token=HF_TOKEN)
+    elif "facebook/xglm" in gen_model_checkpoint or "bigscience/bloomz" in gen_model_checkpoint or "aya-23-8B" in args.gen_model_checkpoint:
+        if load_in_8bit:
+            gen_model = AutoModelForCausalLM.from_pretrained(gen_model_checkpoint, token=HF_TOKEN, device_map="auto",
+                                                             load_in_8bit=True)
+            tokenizer = AutoTokenizer.from_pretrained(gen_model_checkpoint, token=HF_TOKEN, device_map="auto",
+                                                      load_in_8bit=True)
+        else:
+            gen_model = AutoModelForCausalLM.from_pretrained(gen_model_checkpoint, token=HF_TOKEN)
+            tokenizer = AutoTokenizer.from_pretrained(gen_model_checkpoint, token=HF_TOKEN)
+    elif "gpt-3.5-turbo" in gen_model_checkpoint or "gpt-4" in gen_model_checkpoint:
+        gen_model = OpenAI(api_key=OPENAI_TOKEN)
+    elif "command-r" in gen_model_checkpoint:
+        gen_model = cohere.Client(COHERE_TOKEN)
+
+    return gen_model, tokenizer
+

agents/chatgpt.py

@@ -0,0 +1,145 @@
+#%%
+import os
+import re
+
+#%%
+import os
+import json
+import torch
+import numpy as np
+from pathlib import Path
+from transformers import set_seed
+from textgames import GAME_NAMES, LEVEL_IDS, game_filename
+from agents import run_with_agent
+
+#%%
+def set_all_seed(seed=42):
+    set_seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+
+
+#%%
+def _getenv_as_int(attr, default=None):
+    ret = os.getenv(attr, default)
+    return None if ret is None else int(ret)
+
+
+GAME_ST, GAME_ED = _getenv_as_int("TG_GAME_ST", None), _getenv_as_int("TG_GAME_ED", None)
+LVL_ST, LVL_ED = _getenv_as_int("TG_LEVEL_ST", None), _getenv_as_int("TG_LEVEL_ED", '3')
+SID_ST, SID_ED = _getenv_as_int("TG_SID_ST", None), _getenv_as_int("TG_SID_ED", None)
+N_TURNS = _getenv_as_int("TG_N_TURNS", 1)
+ONE_SHOT = bool(int(os.getenv("TG_ONESHOT", "0")))
+GPT_MODEL = os.getenv("TG_GPT_MODEL", "")
+# MAX_NEW_TOKENS = _getenv_as_int("TG_MAX_NEW_TOKENS", 12000)
+
+
+#%%
+def preload_responses():
+    responses_all = dict()
+    for _turn in range(1, N_TURNS+1):
+        fp = os.getenv(
+            f"TG_GPT_OUTPUT_FILE_TURN_{_turn}",
+            (f"model_outputs/__runs__/chatgpt_4o_mini_results/raw/batch_output_chatgpt-{GPT_MODEL}_turn{_turn}"
+             f"{'.1s' if ONE_SHOT else '.zs'}.jsonl")
+        )
+        if not Path(fp).exists():
+            if _turn < N_TURNS:
+                print(f"  batch_output turn {_turn} is not available. path: \"{fp}\"")
+            break
+        with open(fp, "r", encoding="utf8") as i:
+            data = [json.loads(line) for line in i]
+        for d in data:
+            sid, g = d['custom_id'].rsplit('-', 2)[-2:]
+            msg = d['response']['body']['choices'][0]['message']
+            responses_all.setdefault((g, _turn), dict())[sid] = msg['content']
+            responses_all[g, _turn][sid] = msg['content']
+            # assert msg['role'] == 'assistant'
+            # assert msg['refusal'] is None
+        # assert sum(len(v) for v in responses_all.values()) == 24000
+    return responses_all
+RESPONSES_ALL = preload_responses()
+print(f"len(RESPONSES_ALL) = {len(RESPONSES_ALL)}")
+
+
+#%%
+def gpt_postproc(response_txt_batch, *args, **kwargs):
+    response_txt_batch = [response_txt_batch]
+    ret = []
+    for response_txt in response_txt_batch:
+        if response_txt is None:
+            ret.append(response_txt)
+            continue
+        cur = None
+        for pat in [
+            re.compile(r'^```\n?([^`]*)\n?```'),
+            # re.compile(r'\*\*\"?([^\"*]*)\"?\*\*'),
+            re.compile(r'((.|\n)*)\n\nExplanation:\n'),
+        ]:
+            match = pat.search(response_txt)
+            if match:
+                cur = match.group(1).strip()
+                # .replace(" ", "")
+                break
+        ret.append(cur if cur else response_txt)
+    return ret[0]
+
+
+#%%
+def get_gpt_response(texts, game_name, difficulty_level, turn, *args, **kwargs):
+    # global model, tokenizer
+    sid = kwargs['sid']    # sid must be fed as params
+    messages = [
+        ({"role": "user", "content": text}
+         if i % 2 == 0 else
+         {"role": "assistant", "content": text})
+        for i, text in enumerate(texts)
+    ]
+
+    response = None
+    responses_all = RESPONSES_ALL.get((f"{game_filename(game_name)}_{difficulty_level}", turn), {})
+    if responses_all:
+        response = responses_all[sid]
+    elif fp_next := os.getenv("TG_GPT_NEXTTURN_OUTPUT_FILE", None):
+        with open(fp_next, "a", encoding="utf8") as o:
+            o.write(json.dumps({
+                'custom_id': f"{sid}-{game_filename(game_name)}_{difficulty_level}",
+                "method": "POST", "url": "/v1/chat/completions",
+                "body": {
+                    "model": "gpt-4o-mini-2024-07-18",
+                    "max_completion_tokens": 200,
+                    # "messages": [],
+                    'messages': messages,
+                    "seed": 42,
+                    "temperature": 0,
+                }
+            }))
+            o.write("\n")
+
+    return response
+
+
+#%%
+if __name__ == "__main__":
+    fp_out = (f"model_outputs/__runs__/chatgpt_4o_mini_results/process/results_chatgpt-{GPT_MODEL}"
+              f"{'.1s' if ONE_SHOT else '.zs'}"
+              f"{'' if GAME_ST is None else f'.{GAME_ST}'}"
+              f"{'' if LVL_ST is None else f'.{LVL_ST}'}"
+              f".jsonl")
+
+    set_all_seed()
+
+    run_with_agent(
+        fp_out,
+        get_gpt_response,
+        gpt_postproc,
+        n_turns=N_TURNS,
+        game_names_list=GAME_NAMES[GAME_ST:GAME_ED],
+        level_ids_list=LEVEL_IDS[LVL_ST:LVL_ED],
+        sid_indices=(list(map(lambda r: f"session_{r:04}", range(SID_ST or 0, SID_ED or 1000)))
+                     if SID_ST or SID_ED else None),
+        prepend_example=ONE_SHOT,
+        # remove_if_output_file_exist=False,
+        assistant_uses_raw_response=False,
+    )

agents/dsr1_distill.py

@@ -0,0 +1,138 @@
+#%%
+import os
+import re
+
+#%%
+import torch
+import numpy as np
+from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed
+from textgames import THE_GAMES, GAME_NAMES, LEVEL_IDS
+from agents import run_with_agent
+
+#%%
+def set_all_seed(seed=42):
+    set_seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+
+
+#%%
+def _getenv_as_int(attr, default=None):
+    ret = os.getenv(attr, default)
+    return None if ret is None else int(ret)
+
+
+GAME_ST, GAME_ED = _getenv_as_int("TG_GAME_ST", None), _getenv_as_int("TG_GAME_ED", None)
+LVL_ST, LVL_ED = _getenv_as_int("TG_LEVEL_ST", None), _getenv_as_int("TG_LEVEL_ED", '3')
+SID_ST, SID_ED = _getenv_as_int("TG_SID_ST", None), _getenv_as_int("TG_SID_ED", None)
+N_TURNS = _getenv_as_int("TG_N_TURNS", 1)
+ONE_SHOT = bool(int(os.getenv("TG_ONESHOT", "0")))
+MAX_NEW_TOKENS = _getenv_as_int("TG_MAX_NEW_TOKENS", 12000)
+DSR1_SIZE = os.getenv("TG_DSR1_SIZE", "14")    # {1.5, 7, 8, 14, 32, 70}
+DSR1_NAME = {
+    "1.5": "Qwen-1.5",
+    "7": "Qwen-7",
+    "8": "Llama-8",
+    "14": "Qwen-14",
+    "32": "Qwen-32",
+    "70": "Llama-70",
+}
+
+
+#%%
+def dsr1_postproc(response_txt_batch, *args, **kwargs):
+    response_txt_batch = [response_txt_batch]
+    ret = []
+    for response_txt in response_txt_batch:
+        _match = None
+        for pat in [
+            re.compile(r'\\boxed\{([\s\S]*)}'),
+            re.compile(r'</think>\n([\s\S]*)$'),
+            re.compile(r'^```\n?([^`]*)\n?```'),
+        ]:
+            matches = pat.search(response_txt)
+            if matches:
+                _match = matches.group(1).strip()
+                break
+        if _match is not None:
+            ret.append(_match)
+        else:
+            ret.append(response_txt[:256].strip() if response_txt else "")
+    return ret[0]
+
+
+#%%
+def get_dsr1_response(texts_batch, *args, **kwargs):
+    # global model, tokenizer
+    texts_batch = [texts_batch]
+    for texts in texts_batch:
+        if len(texts) > 1 and texts[1].startswith('Correct guess.'):
+            texts[1] = f"\\boxed{{{texts[1]}}}"
+    messages = [
+        [
+            {"role": "user",
+             "content": f"{text}\nPlease reason step by step, and put your final answer within \\boxed{{}} as plain text."}
+            if i % 2 == 0 else
+            {"role": "assistant", "content": {text}}
+            for i, text in enumerate(texts)
+        ]
+        for texts in texts_batch
+    ]
+    text_inputs = tokenizer.apply_chat_template(
+        messages,
+        tokenize=False,
+        add_generation_prompt=True
+    )
+    model_inputs = tokenizer(text_inputs, return_tensors="pt", add_special_tokens=False).to(model.device)
+    output_ids = model.generate(
+        **model_inputs,
+        max_new_tokens=MAX_NEW_TOKENS,
+        do_sample=False,
+        pad_token_id=tokenizer.eos_token_id,
+    )
+    generated_ids = [
+        _output_ids[len(input_ids):] for input_ids, _output_ids in zip(model_inputs.input_ids, output_ids)
+    ]
+    response = [r.strip() for r in tokenizer.batch_decode(generated_ids, skip_special_tokens=True)]
+    return response[0]
+
+
+#%%
+# response = get_dsr1_response(texts)
+# print(dsr1_postproc(response))
+
+
+#%%
+if __name__ == "__main__":
+    fp_out = (f"model_outputs/__runs__/results_deepseek-r1-distill-{DSR1_SIZE}b"
+              f"{'.1s' if ONE_SHOT else '.zs'}"
+              f"{'' if GAME_ST is None else f'.{GAME_ST}'}"
+              f"{'' if LVL_ST is None else f'.{LVL_ST}'}"
+              f".jsonl")
+
+    set_all_seed()
+    model_name = f"deepseek-ai/DeepSeek-R1-Distill-{DSR1_NAME[DSR1_SIZE]}B"
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModelForCausalLM.from_pretrained(
+        model_name,
+        device_map="auto",
+        torch_dtype="auto",
+    )
+    model.generation_config.temperature = None
+    model.generation_config.top_k = None
+    model.generation_config.top_p = None
+
+    run_with_agent(
+        fp_out,
+        get_dsr1_response,
+        dsr1_postproc,
+        n_turns=N_TURNS,
+        game_names_list=GAME_NAMES[GAME_ST:GAME_ED],
+        level_ids_list=LEVEL_IDS[LVL_ST:LVL_ED],
+        sid_indices=(list(map(lambda r: f"session_{r:04}", range(SID_ST or 0, SID_ED or 1000)))
+                     if SID_ST or SID_ED else None),
+        prepend_example=ONE_SHOT,
+        # remove_if_output_file_exist=False,
+        assistant_uses_raw_response=False,
+    )

agents/gemma_2_9b_it.py

@@ -0,0 +1,104 @@
+#%%
+import os
+import re
+
+#%%
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from textgames import THE_GAMES, GAME_NAMES, LEVEL_IDS
+from agents import run_with_agent
+
+
+#%%
+def _getenv_as_int(attr, default=None):
+    ret = os.getenv(attr, default)
+    return None if ret is None else int(ret)
+
+
+GAME_ST, GAME_ED = _getenv_as_int("TG_GAME_ST", None), _getenv_as_int("TG_GAME_ED", None)
+LVL_ST, LVL_ED = _getenv_as_int("TG_LEVEL_ST", None), _getenv_as_int("TG_LEVEL_ED", '3')
+SID_ST, SID_ED = _getenv_as_int("TG_SID_ST", None), _getenv_as_int("TG_SID_ED", None)
+N_TURNS = _getenv_as_int("TG_N_TURNS", 3)
+ONE_SHOT = bool(int(os.getenv("TG_ONESHOT", "0")))
+GEMMA_SIZE = int(os.getenv("TG_GEMMA_SIZE", "9"))    # {3, 9, 27}
+
+
+#%%
+def gemma_postproc(response_txt, game_name, difficulty_level, *args, **kwargs):
+    # if game_name in [THE_GAMES[i] for i in ["1", "7"]]:  # crossword
+    pat = re.compile(r'^```\n?([^`]*)\n?```')
+    match = pat.search(response_txt)
+    if match:
+        return match.group(1).strip().replace(" ", "")
+
+    # elif game_name == THE_GAMES["6"]:  # anagram
+    pat = re.compile(r'\*\*\"?([^\"*]*)\"?\*\*')
+    match = pat.search(response_txt)
+    if match:
+        return match.group(1).strip()
+
+    return response_txt or ""
+
+
+#%%
+def get_gemma_response(texts, game_name, difficulty_level, turn, *args, **kwargs):
+    # global gen_model, tokenizer
+    messages = [
+        {"role": ("model" if i % 2 else "user"), "content": text}
+        for i, text in enumerate(texts)
+    ]
+
+    input_ids = tokenizer.apply_chat_template(
+        messages,
+        add_generation_prompt=True,
+        return_tensors="pt"
+    ).to(gen_model.device)
+
+    terminators = [
+        tokenizer.eos_token_id,
+        tokenizer.convert_tokens_to_ids("<|eot_id|>")
+    ]
+
+    gen_model.generation_config.temperature = None
+    outputs = gen_model.generate(
+        input_ids,
+        max_new_tokens=100,
+        eos_token_id=terminators,
+        do_sample=False,
+        # temperature=.0,
+        # top_p=1,
+    )
+
+    response = outputs[0][input_ids.shape[-1]:]
+    return tokenizer.decode(response, skip_special_tokens=True).strip()
+
+
+#%%
+if __name__ == "__main__":
+    fp_out = (f"model_outputs/results_gemma-2-{GEMMA_SIZE}b-it"
+              f"{'.1s' if ONE_SHOT else '.zs'}"
+              f"{'' if GAME_ST is None else f'.{GAME_ST}'}"
+              f"{'' if LVL_ST is None else f'.{LVL_ST}'}"
+              f".jsonl")
+    gen_model_checkpoint = f"google/gemma-2-{GEMMA_SIZE}b-it"
+
+    quantize = True
+    _kwargs = {
+        "device_map": "auto",
+    } if quantize else {}
+
+    gen_model = AutoModelForCausalLM.from_pretrained(gen_model_checkpoint, **_kwargs)
+    tokenizer = AutoTokenizer.from_pretrained(gen_model_checkpoint, **_kwargs)
+    print(f"    > model.dtype: {gen_model.dtype}")
+
+    run_with_agent(
+        fp_out,
+        get_gemma_response,
+        gemma_postproc,
+        n_turns=N_TURNS,
+        game_names_list=GAME_NAMES[GAME_ST:GAME_ED],
+        level_ids_list=LEVEL_IDS[LVL_ST:LVL_ED],
+        sid_indices=(list(map(lambda r: f"session_{r:04}", range(SID_ST or 0, SID_ED or 1000)))
+                     if SID_ST or SID_ED else None),
+        prepend_example=ONE_SHOT,
+        # remove_if_output_file_exist=False,
+    )

agents/llama3.py

@@ -0,0 +1,102 @@
+#%%
+import os
+import re
+
+#%%
+from transformers import AutoTokenizer, AutoModelForCausalLM
+from textgames import THE_GAMES, GAME_NAMES, LEVEL_IDS
+from agents import run_with_agent
+
+
+#%%
+def _getenv_as_int(attr, default=None):
+    ret = os.getenv(attr, default)
+    return None if ret is None else int(ret)
+
+
+GAME_ST, GAME_ED = _getenv_as_int("TG_GAME_ST", None), _getenv_as_int("TG_GAME_ED", None)
+LVL_ST, LVL_ED = _getenv_as_int("TG_LEVEL_ST", None), _getenv_as_int("TG_LEVEL_ED", '3')
+SID_ST, SID_ED = _getenv_as_int("TG_SID_ST", None), _getenv_as_int("TG_SID_ED", None)
+N_TURNS = _getenv_as_int("TG_N_TURNS", 3)
+ONE_SHOT = bool(int(os.getenv("TG_ONESHOT", "0")))
+LLAMA_SIZE = os.getenv("TG_LLAMA_SIZE", "1-8")
+
+
+#%%
+def llama_postproc(response_txt, *args, **kwargs):
+    # # if game_name in [THE_GAMES[i] for i in ["1", "7"]]:  # crossword
+    # pat = re.compile(r'^```\n?([^`]*)\n?```')
+    # match = pat.search(response_txt)
+    # if match:
+    #     return match.group(1).strip().replace(" ", "")
+    #
+    # # elif game_name == THE_GAMES["6"]:  # anagram
+    # pat = re.compile(r'\*\*\"?([^\"*]*)\"?\*\*')
+    # match = pat.search(response_txt)
+    # if match:
+    #     return match.group(1).strip()
+    return response_txt or ""
+
+
+#%%
+def get_llama_response(texts, *args, **kwargs):
+    # global model, tokenizer
+
+    messages = [
+        # {"role": "system", "content": "You are a bot that responds to weather queries."},
+        *[{"role": ("assistant" if i % 2 else "user"), "content": text} for i, text in enumerate(texts)]
+    ]
+
+    text_inputs = tokenizer.apply_chat_template(
+        messages,
+        tokenize=False,
+        add_generation_prompt=True,
+    )
+    model_inputs = tokenizer([text_inputs], return_tensors="pt").to(model.device)
+
+    model.generation_config.do_sample = False
+    model.generation_config.temperature = None
+    model.generation_config.top_k = None
+    model.generation_config.top_p = None
+    generated_ids = model.generate(
+        **model_inputs,
+        max_new_tokens=128,
+        do_sample=False,
+        pad_token_id=tokenizer.eos_token_id,
+    )
+    generated_ids = [
+        output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
+    ]
+
+    response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+    return response.strip()
+
+
+#%%
+if __name__ == "__main__":
+    fp_out = (f"model_outputs/__runs__/results_llama-3.{LLAMA_SIZE}b-instruct"
+              f"{'.1s' if ONE_SHOT else '.zs'}"
+              f"{'' if GAME_ST is None else f'.{GAME_ST}'}"
+              f"{'' if LVL_ST is None else f'.{LVL_ST}'}"
+              f".jsonl")
+
+    model_name = f"meta-llama/Llama-3.{LLAMA_SIZE}B-Instruct"
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModelForCausalLM.from_pretrained(
+        model_name,
+        device_map="auto",
+        torch_dtype="bfloat16",
+    )
+
+    run_with_agent(
+        fp_out,
+        get_llama_response,
+        llama_postproc,
+        n_turns=N_TURNS,
+        game_names_list=GAME_NAMES[GAME_ST:GAME_ED],
+        level_ids_list=LEVEL_IDS[LVL_ST:LVL_ED],
+        sid_indices=(list(map(lambda r: f"session_{r:04}", range(SID_ST or 0, SID_ED or 1000)))
+                     if SID_ST or SID_ED else None),
+        prepend_example=ONE_SHOT,
+        # remove_if_output_file_exist=False,
+    )

agents/qwen2_5_7b_instruct.py

@@ -0,0 +1,112 @@
+#%%
+import os
+import re
+
+#%%
+import torch
+import numpy as np
+from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed
+from textgames import GAME_NAMES, LEVEL_IDS
+from agents import run_with_agent
+
+
+#%%
+def set_all_seed(seed=42):
+    set_seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+
+
+#%%
+def _getenv_as_int(attr, default=None):
+    ret = os.getenv(attr, default)
+    return None if ret is None else int(ret)
+
+
+GAME_ST, GAME_ED = _getenv_as_int("TG_GAME_ST", None), _getenv_as_int("TG_GAME_ED", None)
+LVL_ST, LVL_ED = _getenv_as_int("TG_LEVEL_ST", None), _getenv_as_int("TG_LEVEL_ED", '3')
+SID_ST, SID_ED = _getenv_as_int("TG_SID_ST", None), _getenv_as_int("TG_SID_ED", None)
+N_TURNS = _getenv_as_int("TG_N_TURNS", 3)
+ONE_SHOT = bool(int(os.getenv("TG_ONESHOT", "0")))
+QWEN_SIZE = int(os.getenv("TG_QWEN_SIZE", "32"))    # {3, 7, 14, 32, 72}  unsupported: {0.5, 1.5}
+
+
+#%%
+def qwen_postproc(response_txt, game_name, difficulty_level, *args, **kwargs):
+    # # if game_name in [THE_GAMES[i] for i in ["1", "7"]]:  # crossword
+    # pat = re.compile(r'^```\n?([^`]*)\n?```')
+    # match = pat.search(response_txt)
+    # if match:
+    #     return match.group(1).strip().replace(" ", "")
+    #
+    # # elif game_name == THE_GAMES["6"]:  # anagram
+    # pat = re.compile(r'\*\*\"?([^\"*]*)\"?\*\*')
+    # match = pat.search(response_txt)
+    # if match:
+    #     return match.group(1).strip()
+    return response_txt or ""
+
+
+#%%
+def get_qwen_response(texts_batch, game_name, difficulty_level, turn, *args, **kwargs):
+    # global model, tokenizer
+    texts_batch = [texts_batch]    # currently we do not support batch
+    messages = [[
+        {"role": "system", "content": "You are Qwen, created by Alibaba Cloud. You are a helpful assistant."},
+        *[{"role": ("assistant" if i % 2 else "user"), "content": text} for i, text in enumerate(texts)]
+    ] for texts in texts_batch ]
+
+    text_inputs = tokenizer.apply_chat_template(
+        messages,
+        tokenize=False,
+        add_generation_prompt=True
+    )
+    model_inputs = tokenizer([text_inputs], return_tensors="pt").to(model.device)
+
+    model.generation_config.temperature = None
+    model.generation_config.top_k = None
+    model.generation_config.top_p = None
+    generated_ids = model.generate(
+        **model_inputs,
+        max_new_tokens=128,
+        do_sample=False,
+    )
+    generated_ids = [
+        output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
+    ]
+
+    response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+    return response.strip()
+
+
+#%%
+if __name__ == "__main__":
+    fp_out = (f"model_outputs/__runs__/results_qwen2-5-{QWEN_SIZE}b-instruct"
+              f"{'.1s' if ONE_SHOT else '.zs'}"
+              f"{'' if GAME_ST is None else f'.{GAME_ST}'}"
+              f"{'' if LVL_ST is None else f'.{LVL_ST}'}"
+              f".jsonl")
+
+    set_all_seed()
+    model_name = f"Qwen/Qwen2.5-{QWEN_SIZE}B-Instruct"
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModelForCausalLM.from_pretrained(
+        model_name,
+        device_map="auto",
+        torch_dtype="auto",
+    )
+    print(f"    > model.dtype: {model.dtype}")
+
+    run_with_agent(
+        fp_out,
+        get_qwen_response,
+        qwen_postproc,
+        n_turns=N_TURNS,
+        game_names_list=GAME_NAMES[GAME_ST:GAME_ED],
+        level_ids_list=LEVEL_IDS[LVL_ST:LVL_ED],
+        sid_indices=(list(map(lambda r: f"session_{r:04}", range(SID_ST or 0, SID_ED or 1000)))
+                     if SID_ST or SID_ED else None),
+        prepend_example=ONE_SHOT,
+        # remove_if_output_file_exist=False,
+    )

agents/qwen2_5_math.py

@@ -0,0 +1,137 @@
+#%%
+import os
+import re
+
+#%%
+import torch
+import numpy as np
+from transformers import AutoTokenizer, AutoModelForCausalLM, set_seed, BitsAndBytesConfig
+from textgames import GAME_NAMES, LEVEL_IDS
+from agents import run_with_agent
+
+
+#%%
+def set_all_seed(seed=42):
+    set_seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+
+
+#%%
+def _getenv_as_int(attr, default=None):
+    ret = os.getenv(attr, default)
+    return None if ret is None else int(ret)
+
+
+GAME_ST, GAME_ED = _getenv_as_int("TG_GAME_ST", None), _getenv_as_int("TG_GAME_ED", None)
+LVL_ST, LVL_ED = _getenv_as_int("TG_LEVEL_ST", None), _getenv_as_int("TG_LEVEL_ED", '3')
+SID_ST, SID_ED = _getenv_as_int("TG_SID_ST", None), _getenv_as_int("TG_SID_ED", None)
+N_TURNS = _getenv_as_int("TG_N_TURNS", 3)
+ONE_SHOT = bool(int(os.getenv("TG_ONESHOT", "0")))
+# MAX_NEW_TOKENS = _getenv_as_int("TG_MAX_NEW_TOKENS", 4096)
+QWEN_MATH_SIZE = os.getenv("TG_QWEN_MATH_SIZE", "7")    # {1.5, 7, 72}
+QUANTIZE = _getenv_as_int("TG_QUANTIZE", 4)
+
+
+#%%
+def qwenmath_postproc(response_txt_batch, *args, **kwargs):
+    response_txt_batch = [response_txt_batch]
+    ret = []
+    for response_txt in response_txt_batch:
+        _match = None
+        for pat in [
+            re.compile(r'\\boxed\{([\s\S]*)}'),
+            re.compile(r'^```\n?([^`]*)\n?```'),
+            # re.compile(r'</think>\n([\s\S]*)$'),
+        ]:
+            matches = pat.search(response_txt)
+            if matches:
+                _match = matches.group(1).strip()
+                break
+        if _match is not None:
+            ret.append(_match)
+        else:
+            ret.append(response_txt if response_txt else "")
+    return ret[0]
+
+
+#%%
+def get_qwenmath_response(texts_batch, *args, **kwargs):
+    # global model, tokenizer
+    texts_batch = [texts_batch]
+    for texts in texts_batch:
+        if (len(texts) > 1) and texts[2].startswith('Correct guess.'):    # assert len(texts) % 2 == 1
+            texts[1] = f"\\boxed{{{texts[1]}}}"
+    messages = [
+        [
+            {"role": "system", "content": "Please reason step by step, and put your final answer within \\boxed{} as plain text."},
+            *[{"role": ("user" if i % 2 == 0 else "assistant"), "content": text} for i, text in enumerate(texts)],
+        ]
+        for texts in texts_batch
+    ]
+    # print(f"\n{messages[0]}", end="\n=====\n\n")
+
+    text_inputs = tokenizer.apply_chat_template(
+        messages,
+        tokenize=False,
+        add_generation_prompt=True
+    )
+    model_inputs = tokenizer(text_inputs, return_tensors="pt", add_special_tokens=False).to(model.device)
+
+    generated_ids = model.generate(
+        **model_inputs,
+        max_new_tokens=512,
+        do_sample=False,
+    )
+    generated_ids = [
+        output_ids[len(input_ids):] for input_ids, output_ids in zip(model_inputs.input_ids, generated_ids)
+    ]
+
+    response = tokenizer.batch_decode(generated_ids, skip_special_tokens=True)[0]
+    return response.strip()
+
+
+#%%
+if __name__ == "__main__":
+    fp_out = (f"model_outputs/__runs__/results_qwen2-5-math-{QWEN_MATH_SIZE}b-instruct_{QUANTIZE}bit"
+              f"{'.1s' if ONE_SHOT else '.zs'}"
+              f"{'' if GAME_ST is None else f'.{GAME_ST}'}"
+              f"{'' if LVL_ST is None else f'.{LVL_ST}'}"
+              f".jsonl")
+
+    set_all_seed()
+    if QWEN_MATH_SIZE in ['72'] and QUANTIZE < 16:
+        _additional_kwargs = {
+            "quantization_config": (
+                BitsAndBytesConfig(load_in_8bit=True)
+                if QUANTIZE == 8 else
+                BitsAndBytesConfig(load_in_4bit=True)
+            ),
+            "low_cpu_mem_usage": True,
+        }
+    else:
+        _additional_kwargs = {"device_map": "auto"}
+
+    model_name = f"Qwen/Qwen2.5-Math-{QWEN_MATH_SIZE}B-Instruct"
+    tokenizer = AutoTokenizer.from_pretrained(model_name)
+    model = AutoModelForCausalLM.from_pretrained(
+        model_name,
+        torch_dtype="auto",
+        **_additional_kwargs,
+    )
+    print(f"    > model.dtype: {model.dtype}")
+
+    run_with_agent(
+        fp_out,
+        get_qwenmath_response,
+        qwenmath_postproc,
+        n_turns=N_TURNS,
+        game_names_list=GAME_NAMES[GAME_ST:GAME_ED],
+        level_ids_list=LEVEL_IDS[LVL_ST:LVL_ED],
+        sid_indices=(list(map(lambda r: f"session_{r:04}", range(SID_ST or 0, SID_ED or 1000)))
+                     if SID_ST or SID_ED else None),
+        prepend_example=ONE_SHOT,
+        # remove_if_output_file_exist=False,
+        assistant_uses_raw_response=True,
+    )

agents/runner.py

@@ -0,0 +1,89 @@
+#%%
+import os
+import json
+
+from textgames import GAME_NAMES, LEVEL_IDS, game_filename, _game_class_from_name
+
+from tqdm import tqdm
+from itertools import product
+from pathlib import Path
+from typing import Union, Callable
+
+
+def response_postprocess(response_txt, game_name, difficulty_level):
+    return response_txt or ""
+
+
+def run_with_agent(fp_out: Union[str, Path],
+                   get_response: Callable,
+                   get_postprocess: Callable = response_postprocess,
+                   n_turns=3,
+                   game_names_list=GAME_NAMES,
+                   level_ids_list=LEVEL_IDS[:3],
+                   sid_indices=None,  # sid_index_range=range(0, 1000),
+                   remove_if_output_file_exist=True,
+                   prepend_example=False,
+                   assistant_uses_raw_response=True,
+                   ) -> None:
+    os.makedirs(os.path.dirname(os.path.abspath(fp_out)), exist_ok=True)
+    print(fp_out)
+    if remove_if_output_file_exist:
+        with open(fp_out, "wb"):
+            pass
+
+    for game_name, difficulty_level in product(game_names_list, level_ids_list):
+        game_str = f"{game_filename(game_name)}_{difficulty_level}"
+        game_cls = _game_class_from_name(game_name)
+        with open(f"problemsets/{game_str}.json", "r", encoding="utf8") as f:
+            sid_prompt_dict = json.load(f)
+        if sid_indices is not None:
+            sid_prompt_dict = {k: sid_prompt_dict[k] for k in sid_indices}
+
+        correct_cnt, exception_cnt = 0, 0
+        for sid, prompt in tqdm(sid_prompt_dict.items(), desc=game_str, total=len(sid_prompt_dict)):
+            cur_game = game_cls()
+            cur_game.load_game(prompt)
+            if prepend_example:
+                texts = [*cur_game.example(), f"Correct guess. Now let's try another example.\n{cur_game.get_prompt()}"]
+            else:
+                texts = [cur_game.get_prompt()]
+            for turn in range(1, n_turns + 1):
+                response_raw, response, e = None, None, None
+                solved, val_msg = False, None
+                try:
+                    response_raw = get_response(texts, game_name, difficulty_level, turn, sid=sid)
+                    response = get_postprocess(response_raw, game_name, difficulty_level)
+                    texts.append(response_raw if assistant_uses_raw_response else response)
+                    solved, val_msg = (False, None) if response is None else cur_game.validate(response)
+                    texts.append(
+                        f"Bad guess (Wrong Answer).\n{val_msg}\nPlease try again and print the answer only."
+                        if not solved else "Correct guess."
+                    )
+                except Exception as _e:
+                    e = _e
+                    # print(e)
+                # assert False, {"texts": texts, "response": response_raw,
+                #                "args": (n_turns, game_names_list, remove_if_output_file_exist, prepend_example, assistant_uses_raw_response)}
+                with open(fp_out, "a", encoding="utf8") as o:
+                    json.dump({
+                        "game": game_str,
+                        "session": sid,
+                        "turn": turn,
+                        "response": response,
+                        "solved": solved,
+                        "val_msg": val_msg,
+                        "response_raw": response_raw,
+                        "error": repr(e) if e else e,
+                    }, o, ensure_ascii=False)
+                    o.write("\n")
+                if solved:
+                    correct_cnt += 1
+                if e:
+                    exception_cnt += 1
+                if solved or e:
+                    break
+
+        print(f"{game_filename(game_name)}_-_{difficulty_level}")
+        print(f"    > Correct: {correct_cnt:>6,}  ({correct_cnt / len(sid_prompt_dict):.2%})")
+        print(f"    > Error  : {exception_cnt:>6,}  ({exception_cnt / len(sid_prompt_dict):.2%})")
+

play_gradio.py

@@ -51,7 +51,7 @@ def greet(request: gr.Request):
 
 #%%
 with gr.Blocks(title="TextGames", css=css, delete_cache=(3600, 3600)) as demo:
-    ((m, logout_btn, solved_games_df, game_radio, level_radio, new_game_btn, render_toggle),
+    ((m, logout_btn, solved_games_df, game_radio, level_radio, new_game_btn, render_toggle, reset_sid_btn),
      (session_state, is_solved, solved_games, user_state, uid_state),
      ) = declare_components(demo, greet)
 
@@ -64,7 +64,7 @@ demo.launch(
     auth=file_based_auth,
     favicon_path=favicon_path if os.path.exists(favicon_path) else None,
     share=True,
-    ssr_mode=False,
+    show_api=False,
 )
 
 

play_helper.py

@@ -1,6 +1,7 @@
 # %%
 import os
 import time
+import json
 import pandas as pd
 import gradio as gr
 import hashlib
@@ -19,19 +20,27 @@ from googleapiclient.discovery import build
 from googleapiclient.errors import HttpError
 from googleapiclient.http import MediaFileUpload, MediaIoBaseDownload
 
+# %%
+_leaderboards = f"{os.getenv('TEXTGAMES_OUTPUT_DIR', '.')}/_leaderboards.jsonl"
+
 
 # %%
-def declare_components(demo, greet):
+def declare_components(demo, greet, use_login_button=False):
     with gr.Row():
         with gr.Column(scale=1):
             m = gr.Markdown("Welcome to TextGames!", elem_id="md-greeting")
-            logout_btn = gr.Button("Logout", link="/logout", variant='huggingface', size='sm', elem_id="btn-logout")
+            if use_login_button:
+                logout_btn = gr.LoginButton(size='sm')
+                reset_sid_btn = gr.Button("♻️ Reset Game Progress", variant='huggingface', size='sm')
+            else:
+                logout_btn = gr.Button("Logout", link="/logout", variant='huggingface', size='sm', elem_id="btn-logout")
+                reset_sid_btn = gr.Button(interactive=False, visible=False, size='sm')
         with gr.Column(scale=2):
-            solved_games_df = gr.DataFrame(headers=[g.split('\t', 1)[0] for g in GAME_NAMES], label="Finished Games",
-                                           interactive=False, elem_id="df-solved-games")
-    game_radio = gr.Radio(GAME_NAMES, label="Game", elem_id="radio-game-name")
-    level_radio = gr.Radio(LEVELS, label="Level", elem_id="radio-level-name")
-    new_game_btn = gr.Button("Start Game", elem_id="btn-start-game")
+            solved_games_df = gr.DataFrame(headers=[g.split('\t', 1)[0] for g in GAME_NAMES], label="Attempted Games",
+                                           row_count=(1, 'fixed'), interactive=False, elem_id="df-solved-games")
+    level_radio = gr.Radio(LEVELS, label="Level", elem_id="radio-level-name", visible=False)
+    game_radio = gr.Radio(GAME_NAMES, label="Game", elem_id="radio-game-name", visible=False)
+    new_game_btn = gr.Button("Start Game", elem_id="btn-start-game", visible=False)
     render_toggle = gr.Checkbox(False, visible=False, interactive=False)
 
     # cur_game_start = gr.BrowserState()
@@ -41,9 +50,12 @@ def declare_components(demo, greet):
     user_state = gr.State()
     uid_state = gr.State()
 
+    if not os.path.exists(_leaderboards):
+        download_from_drive(_leaderboards, compare_checksum=False)
+
     session_state.change(
-        lambda s: session_state_change_fn(s, 2, 0, 2, 0),
-        [session_state], [game_radio, level_radio, new_game_btn, logout_btn], js=js_remove_input_helper,
+        lambda s: session_state_change_fn(s, 2, 0, 3, 0),
+        [session_state], [game_radio, level_radio, new_game_btn, logout_btn, reset_sid_btn], js=js_remove_input_helper,
     )
     new_game_btn.click(check_to_start_new_game, [game_radio, level_radio, user_state, uid_state], [session_state])
     solved_games.change(solved_games_change_fn, solved_games, solved_games_df)
@@ -54,13 +66,15 @@ def declare_components(demo, greet):
     ).then(
         lambda: gr.update(interactive=False), None, [new_game_btn],
     ).then(
-        check_played_game, [solved_games, uid_state], [solved_games, solved_games_df]
+        check_played_game, [solved_games, user_state], [solved_games, solved_games_df]
     ).then(
-        lambda: gr.update(interactive=True), None, [new_game_btn],
+        lambda uid: ([gr.update(visible=True, interactive=True)] if uid else
+                     [gr.update(visible=False, interactive=False)]) * 3,
+        [uid_state], [level_radio, game_radio, new_game_btn]
     )
 
     return (
-        (m, logout_btn, solved_games_df, game_radio, level_radio, new_game_btn, render_toggle),
+        (m, logout_btn, solved_games_df, game_radio, level_radio, new_game_btn, render_toggle, reset_sid_btn),
         (session_state, is_solved, solved_games, user_state, uid_state),
     )
 
@@ -489,7 +503,8 @@ def _is_checksum_same(fp_out, matches=None, mime_type="application/octet-stream"
         matches = _files.list(
             q=f"'{_folder_id}' in parents and mimeType='{mime_type}' and name = '{fp_out.rsplit('/', 1)[-1]}'",
             fields=f"files(name, id, {_cksm_methods_str})",
-        ).execute()['files']
+        ).execute()
+        matches = matches['files']
     if not os.path.exists(fp_out):
         return None, None, matches
     with open(fp_out, "rb") as o:
@@ -502,9 +517,9 @@ def _is_checksum_same(fp_out, matches=None, mime_type="application/octet-stream"
 
 
 # %%
-def upload_to_drive(fp_out, matches=None, mime_type="application/octet-stream", compare_checksum=True):
+def upload_to_drive(fp_out, matches=None, mime_type="application/octet-stream", compare_checksum=True, update=False):
     if compare_checksum:
-        same_checksum, _, _ = _is_checksum_same(fp_out, matches, mime_type)
+        same_checksum, _, matches = _is_checksum_same(fp_out, matches, mime_type)
         # same_checksum, _, _ = _is_checksum_same(
         #     fp_out, **{k: v for k, v in [('matches', matches), ('mime_type', mime_type)] if v})
         if same_checksum:
@@ -513,7 +528,11 @@ def upload_to_drive(fp_out, matches=None, mime_type="application/octet-stream",
     file_metadata = {"name": fn, "parents": [_folder_id]}
     media = MediaFileUpload(fp_out)
     try:
-        _files.create(body=file_metadata, media_body=media).execute()
+        if update and matches:
+            file_metadata.pop("parents")
+            _files.update(fileId=matches[0]['id'], body=file_metadata, media_body=media).execute()
+        else:
+            _files.create(body=file_metadata, media_body=media).execute()
     except HttpError as error:
         msg = f"Failed to upload the file, error: {error}"
         print(msg)
@@ -547,7 +566,7 @@ def download_from_drive(fp_out, matches=None, mime_type="application/octet-strea
 
 # %%
 def start_new_game(game_name, level, session_state_component, is_solved_component, solved_games_component,
-                   user=None, show_timer=False, uid=None):
+                   user=None, show_timer=False, uid=None, sid=None):
     # cur_game_id = GAME_IDS[GAME_NAMES.index(game_name)]
     difficulty_level = LEVEL_IDS[LEVELS.index(level)]
 
@@ -555,11 +574,16 @@ def start_new_game(game_name, level, session_state_component, is_solved_componen
     #     elapsed_text = gr.Textbox("N/A", label=f"{game_name}", info=f"{level}", )
     #     gr.Timer(.3).tick(_calc_time_elapsed, [cur_game_start, elapsed_text, is_solved_component], [elapsed_text])
 
-    fp_out = _get_file_output(game_name, difficulty_level, uid)
+    if (not sid) and user and ('sid' in user):
+        sid = user['sid']
+
+    fp_out = _get_file_output(game_name, difficulty_level, f"{uid}_{sid}")
     cur_game = (
         new_game(game_name, difficulty_level)
         if user is None else
         preload_game(game_name, difficulty_level, user)
+        if sid is None else
+        preload_game(game_name, difficulty_level, user, sid=sid)
     )
     cur_game.attach_stats_output_(fp_out)
     cur_game.flush_stats_(user_info_to_flush=user)
@@ -616,8 +640,12 @@ def start_new_game(game_name, level, session_state_component, is_solved_componen
                      js=js_submit)
     give_up_checkbox = gr.Checkbox(False, visible=False, interactive=False)
     give_up_btn.click(
+        lambda: (gr.update(interactive=False), gr.update(interactive=False)), None, [submit_btn, give_up_btn]
+    ).then(
         lambda x: x, [give_up_checkbox], [give_up_checkbox],
         js="(x) => confirm('🥹 Give-up? 💸')"
+    ).then(
+        lambda: (gr.update(interactive=True), gr.update(interactive=True)), None, [submit_btn, give_up_btn]
     )
 
     def _forfeiting(confirmed, _solved_games):
@@ -640,6 +668,8 @@ def start_new_game(game_name, level, session_state_component, is_solved_componen
     def game_is_solved(_is_solved, _session_state, _solved_games, progress=gr.Progress()):
         if _is_solved:
             if level in LEVELS and level not in _solved_games[game_name]:
+                if isinstance(_solved_games[game_name], str):
+                    _solved_games[game_name] = []
                 _solved_games[game_name].append(level)
             return (
                 2,
@@ -655,8 +685,16 @@ def start_new_game(game_name, level, session_state_component, is_solved_componen
 
     def finalize_game(_is_solved):
         if _is_solved:
-            gr.Info("Reporting... Please click the button when available...")
+            gr.Info(f"Wrapping things up... Please click the button when available...<br/>"
+                    f"Time: {cur_game.end_timestamp-cur_game.start_timestamp:4.1f} sec. Attempt: {cur_game.attempt_count}.")
+            with open(_leaderboards, "a", encoding="utf-8") as f:
+                json.dump({'uid': uid, 'sid': sid, 'turns': cur_game.attempt_count,
+                           'st': cur_game.start_timestamp, 'ed': cur_game.end_timestamp,
+                           'game_name': game_name, 'difficulty_level': difficulty_level,
+                           }, f)
+                f.write("\n")
             upload_to_drive(fp_out)
+            upload_to_drive(_leaderboards, update=True)
             return gr.update(interactive=True)
         return gr.update()
 
@@ -673,13 +711,14 @@ def start_new_game(game_name, level, session_state_component, is_solved_componen
 
 
 # %%
-def check_to_start_new_game(game_name, level, user=None, uid=None):
-    print(game_name, level)
+def check_to_start_new_game(game_name, level, user=None, uid=None, sid=None):
+    print(game_name, level, uid, sid)
     if game_name is None or level is None:
         raise gr.Error("please choose both Game & Level")
-    fp = _get_file_output(game_name, LEVEL_IDS[LEVELS.index(level)], uid)
+    fp = _get_file_output(game_name, LEVEL_IDS[LEVELS.index(level)], f"{uid}_{sid}")
     if os.path.exists(fp):
-        raise gr.Error(f"You have done this game already.<br/>{game_name} - {level}")
+        # raise gr.Error(f"You have done this game already.<br/>{game_name} - {level}")
+        gr.Warning("You have done this game already. Only first attempt is recorded in the scoreboard.")
     if user is None:
         gr.Warning("no user, game will be generated randomly")
     # else:
@@ -691,16 +730,19 @@ def check_to_start_new_game(game_name, level, user=None, uid=None):
 
 
 # %%
-def check_played_game(solved_games, uid, progress=gr.Progress()):
+def check_played_game(solved_games, user, progress=gr.Progress()):
+    uid = user['email']
+    sid = user.get('sid', None)
     matches = _files.list(
         q=f"'{_folder_id}' in parents and mimeType='application/octet-stream' and name contains '{uid}_-_'",
         fields=f"files(name, id, {_cksm_methods_str})",
-    ).execute()['files']
+    ).execute()
+    matches = matches['files']
     ret = dict()
     for game_name in solved_games.keys():
         cur = []
         for level, level_id in zip(LEVELS, LEVEL_IDS):
-            fp_out = _get_file_output(game_name, level_id, uid)
+            fp_out = _get_file_output(game_name, level_id, f"{uid}_{sid}")
             _matches = list(filter(lambda m: fp_out.endswith(m['name']), matches))
             if os.path.exists(fp_out):
                 upload_to_drive(fp_out, _matches)
@@ -708,7 +750,7 @@ def check_played_game(solved_games, uid, progress=gr.Progress()):
                 download_from_drive(fp_out, _matches)
             if os.path.exists(fp_out):
                 cur.append(level)
-        ret[game_name] = cur
+        ret[game_name] = cur or '∅'
     return ret, gr.update()
 
 

play_with_auth.py

@@ -130,7 +130,7 @@ with gr.Blocks(title="TextGames") as login_demo:
 app = gr.mount_gradio_app(app, login_demo, path="/login")
 
 with gr.Blocks(title="TextGames", css=css, delete_cache=(3600, 3600)) as demo:
-    ((m, logout_btn, solved_games_df, game_radio, level_radio, new_game_btn, render_toggle),
+    ((m, logout_btn, solved_games_df, game_radio, level_radio, new_game_btn, render_toggle, reset_sid_btn),
      (session_state, is_solved, solved_games, user_state, uid_state),
      ) = declare_components(demo, greet)
 

play_with_hf.py

@@ -0,0 +1,132 @@
+from __future__ import annotations
+
+#%%
+import os
+# os.environ.setdefault("GRADIO_SERVER_PORT", "1080")
+# os.environ.setdefault("TEXTGAMES_SHOW_HIDDEN_LEVEL", "1")
+os.environ.setdefault("TEXTGAMES_LOADGAME_DIR", "problemsets")
+os.environ.setdefault("TEXTGAMES_LOADGAME_ID", "42")
+os.environ.setdefault("TEXTGAMES_MOCKUSER", "")
+os.environ.setdefault("TEXTGAMES_OUTPUT_DIR", "user_outputs")
+favicon_path = "textgames-scrabble-black2-ss.png"
+
+#%%
+from play_helper import css, declare_components, start_new_game, check_played_game, download_from_drive, upload_to_drive, _leaderboards
+import pandas as pd
+import gradio as gr
+import random
+import json
+from textgames import GAME_NAMES
+
+
+#%%
+os.makedirs(os.getenv('TEXTGAMES_OUTPUT_DIR', '.'), exist_ok=True)
+
+
+#%%
+def generate_sid(fp):
+    rand_int = random.randint(0, 1000)
+    with open(fp, "w", encoding="utf8") as f:
+        f.write(f"session_{rand_int:04}\n")
+    upload_to_drive(fp, mime_type="text/plain", update=True)
+
+
+#%%
+def get_sid(uid, force_generate_sid=False):
+    fp = f"{os.getenv('TEXTGAMES_OUTPUT_DIR')}/{uid}_sid.txt"
+    if force_generate_sid:
+        generate_sid(fp)
+    if not os.path.exists(fp):
+        download_from_drive(fp, mime_type="text/plain", compare_checksum=False)
+    if not os.path.exists(fp):
+        generate_sid(fp)
+    with open(fp, "r", encoding="utf8") as f:
+        sid = [_ for _ in f][-1]
+    return sid.strip()
+
+
+#%%
+def greet(request: gr.OAuthProfile | None):
+    user = {'email': os.getenv('TEXTGAMES_MOCKUSER', ''), 'name': ""}
+    if request is not None:
+        user = {'email': request.username, 'name': request.name, 'sid': get_sid(request.username)}
+    return f"""
+        Welcome to TextGames, {user['name'] or 'please login'}!
+    """, user, user['email']
+
+
+#%%
+with gr.Blocks(title="TextGames", css=css, delete_cache=(3600, 3600)) as demo:
+    ((m, logout_btn, solved_games_df, game_radio, level_radio, new_game_btn, render_toggle, reset_sid_btn),
+     (session_state, is_solved, solved_games, user_state, uid_state),
+     ) = declare_components(demo, greet, use_login_button=True)
+    logout_btn.activate()
+
+    reset_sid_checkbox = gr.Checkbox(False, visible=False, interactive=False)
+    reset_sid_btn.click(
+        lambda: [gr.update(interactive=False)]*2, None, [reset_sid_btn, new_game_btn]
+    ).then(
+        lambda x: x, [reset_sid_checkbox], [reset_sid_checkbox],
+        js="(x) => confirm('Reset Progress? (cannot be undone)')"
+    ).then(
+        lambda: [gr.update(interactive=True)]*2, None, [reset_sid_btn, new_game_btn]
+    )
+
+    def _resetting(confirmed, user):
+        uid = user.get('email', None) if isinstance(user, dict) else None
+        if uid is None:
+            gr.Warning("You need to log in first!")
+        elif confirmed:
+            user['sid'] = get_sid(uid, force_generate_sid=True)
+        return user, False
+    reset_sid_checkbox.change(
+        lambda: [gr.update(interactive=False)]*3, None, [logout_btn, reset_sid_btn, new_game_btn]
+    ).then(
+        _resetting, [reset_sid_checkbox, user_state], [user_state, reset_sid_checkbox]
+    ).then(
+        check_played_game, [solved_games, user_state], [solved_games, solved_games_df]
+    ).then(
+        lambda: [gr.update(interactive=True)]*3, None, [logout_btn, reset_sid_btn, new_game_btn]
+    )
+
+
+    @gr.render(inputs=[game_radio, level_radio, user_state, session_state, uid_state], triggers=[render_toggle.change])
+    def _start_new_game(game_name, level, user, _session_state, _uid_state):
+        if _session_state in [1, 2]:
+            start_new_game(game_name, level, session_state, is_solved, solved_games, user=user, uid=_uid_state)
+
+#%%
+with demo.route("Leaderboards", "/leaderboard") as demo_leaderboard:
+    gr.Markdown("Under Construction. Will be available soon.")
+    leaderboards = []
+    for tab in ["🚅 Easy", "🚀 Medium", "🛸 Hard"]:
+        with gr.Tab(tab):
+            leaderboards.append(gr.DataFrame(label="Rankings"))
+
+    # if os.path.exists(_leaderboards):
+    #     datas = []
+    #     with open(_leaderboards, "r", encoding="utf8") as f:
+    #         for line in f:
+    #             datas.append(json.loads(line))
+    #     concat = [{'Level': d['difficulty_level'], 'User': d['uid'], 'Game': d['game_name'].split('\t', 1)[0], 'Attempts': d['turns'],
+    #                "Time": d['ed'] - d['st']} for d in datas]
+    # else:
+    def add_dummies():
+        return pd.DataFrame({
+            'User': ['dummy'],
+            'Solved': [' '.join([g.split('\t', 1)[0] for g in GAME_NAMES])],
+            'Attempts': [8],
+            'Time': [7200.8],
+        })
+    for l in leaderboards:
+        demo_leaderboard.load(add_dummies, None, [l])
+
+
+#%%
+# demo.launch()
+demo.launch(
+    favicon_path=favicon_path if os.path.exists(favicon_path) else None,
+    show_api=False,
+)
+
+

reval_ana3.py

@@ -0,0 +1,87 @@
+import os
+import json
+from tqdm import tqdm
+from textgames import GAME_NAMES, game_filename, _game_class_from_name
+from pathlib import Path
+
+GAME_NAME = GAME_NAMES[5]
+PROBLEMSET_DIR = Path(os.getenv("TG_PROBLEMSET_DIR", "problemsets"))
+MODEL_OUTPUT_DIR = Path(os.getenv("TG_MODEL_OUTPUT_DIR", "model_outputs"))
+OUTPUT_FILENAMES = [
+    # "results_gemma-2-9b-it.1s.jsonl",
+    # "results_gemma-2-9b-it.zs.jsonl",
+    # "results_gemma-2-27b-it.1s.jsonl",
+    # "results_gemma-2-27b-it.zs.jsonl",
+    #
+    # "results_llama-3.1-8b-instruct.1s.jsonl",
+    # "results_llama-3.1-8b-instruct.zs.jsonl",
+    # "results_llama-3.1-70b-instruct.1s.jsonl",
+    # "results_llama-3.1-70b-instruct.zs.jsonl",
+    # "results_llama-3.3-70b-instruct.1s.jsonl",
+    # "results_llama-3.3-70b-instruct.zs.jsonl",
+    #
+    # "results_qwen2-5-7b-instruct.1s.jsonl",
+    # "results_qwen2-5-7b-instruct.zs.jsonl",
+    # "results_qwen2-5-14b-instruct.1s.jsonl",
+    # "results_qwen2-5-14b-instruct.zs.jsonl",
+    # "results_qwen2-5-32b-instruct.1s.jsonl",
+    # "results_qwen2-5-32b-instruct.zs.jsonl",
+    # "results_qwen2-5-72b-instruct.1s.jsonl",
+    # "results_qwen2-5-72b-instruct.zs.jsonl",
+    #
+    # "results_deepseek-r1-distill-14b.1s.jsonl",
+    # "results_deepseek-r1-distill-14b.zs.jsonl",
+    # "results_deepseek-r1-distill-14b.rerun.1s.jsonl",
+    #
+    # "results_chatgpt-4o-mini.zs.jsonl",
+    # "results_chatgpt-o3-mini.zs.jsonl",
+    #
+    # "results_qwen2-5-7b-instruct_sp.1s.jsonl",
+    # "results_qwen2-5-7b-instruct_sp.zs.jsonl",
+
+    # "results_deepseek-r1-distill-8b.1s.jsonl",
+    "results_deepseek-r1-distill-8b.zs.jsonl",
+]
+
+# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+# !!! Must run reval_bracket_rerun.py first !!!
+# !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+
+def revalidate_anagram_3(fp, reval_dir="revalidate_anagram_3", source_dir="prior_revalidate"):
+    os.makedirs(MODEL_OUTPUT_DIR/reval_dir, exist_ok=True)
+    count_pos, count_neg = 0, 0
+    with (open(MODEL_OUTPUT_DIR/source_dir/fp, "r", encoding="utf8") as i,
+          open(MODEL_OUTPUT_DIR/reval_dir/fp, "w", encoding="utf8") as o,
+          tqdm(total=1000, desc=fp) as pbar,
+          ):
+        for line in i:
+            res = json.loads(line)
+            if (res['game'] == f"{game_filename(GAME_NAME)}_3"):
+                if (res['turn'] == 1):
+                    cur_sid = res["session"]
+                    prompt = sid_prompt_dict[cur_sid]
+                    cur_game = game_cls()
+                    cur_game.load_game(prompt)
+                    pbar.update(1)
+                elif solved == True:
+                    continue
+                else:
+                    assert cur_sid == res["session"]
+                solved, _ = cur_game.validate(res["response"])
+                if solved and not res["solved"]:
+                    count_pos += 1
+                elif not solved and res["solved"]:
+                    count_neg += 1
+                res["solved"] = solved
+            o.write(json.dumps(res))
+            o.write("\n")
+    return count_pos, count_neg
+
+
+if __name__ == "__main__":
+    game_cls = _game_class_from_name(GAME_NAME)
+    with open(f"{PROBLEMSET_DIR}/{game_filename(GAME_NAME)}_3.json", "r", encoding="utf8") as f:
+        sid_prompt_dict = json.load(f)
+    for fp in OUTPUT_FILENAMES:
+        print(revalidate_anagram_3(fp))

reval_bracket_all.py

@@ -0,0 +1,94 @@
+import os
+import json
+from tqdm import tqdm
+from textgames import GAME_NAMES, game_filename, _game_class_from_name
+from pathlib import Path
+
+GAME_NAME = GAME_NAMES[6]
+PROBLEMSET_DIR = Path(os.getenv("TG_PROBLEMSET_DIR", "problemsets"))
+MODEL_OUTPUT_DIR = Path(os.getenv("TG_MODEL_OUTPUT_DIR", "model_outputs"))
+OUTPUT_FILENAMES = [
+    # "results_gemma-2-9b-it.1s.jsonl",
+    # "results_gemma-2-9b-it.zs.jsonl",
+    # "results_gemma-2-27b-it.1s.jsonl",
+    # "results_gemma-2-27b-it.zs.jsonl",
+    #
+    # "results_llama-3.1-8b-instruct.1s.jsonl",
+    # "results_llama-3.1-8b-instruct.zs.jsonl",
+    # "results_llama-3.1-70b-instruct.1s.jsonl",
+    # "results_llama-3.1-70b-instruct.zs.jsonl",
+    # "results_llama-3.3-70b-instruct.1s.jsonl",
+    # "results_llama-3.3-70b-instruct.zs.jsonl",
+    #
+    # "results_qwen2-5-7b-instruct.1s.jsonl",
+    # "results_qwen2-5-7b-instruct.zs.jsonl",
+    # "results_qwen2-5-14b-instruct.1s.jsonl",
+    # "results_qwen2-5-14b-instruct.zs.jsonl",
+    # "results_qwen2-5-32b-instruct.1s.jsonl",
+    # "results_qwen2-5-32b-instruct.zs.jsonl",
+    # "results_qwen2-5-72b-instruct.1s.jsonl",
+    # "results_qwen2-5-72b-instruct.zs.jsonl",
+    #
+    # "results_deepseek-r1-distill-14b.1s.jsonl",
+    # "results_deepseek-r1-distill-14b.zs.jsonl",
+    # "results_deepseek-r1-distill-14b.rerun.1s.jsonl",
+    #
+    # "results_chatgpt-4o-mini.1s.jsonl",
+    # "results_chatgpt-4o-mini.zs.jsonl",
+    # "results_chatgpt-o3-mini.zs.jsonl",
+    #
+    # "results_qwen2-5-7b-instruct_sp.1s.jsonl",
+    # "results_qwen2-5-7b-instruct_sp.zs.jsonl",
+
+    # "results_deepseek-r1-distill-8b.1s.jsonl",
+    "results_deepseek-r1-distill-8b.zs.jsonl",
+]
+
+
+def revalidate_bracket(fp, reval_dir="revalidate_bracket_all",
+                       source_dirs=("revalidate_bracket_rerun", "revalidate_anagram_3",)):
+    os.makedirs(MODEL_OUTPUT_DIR/reval_dir, exist_ok=True)
+    count_pos, count_neg = 0, 0
+    source_dir = "."
+    for source_dir in source_dirs:
+        if (MODEL_OUTPUT_DIR / source_dir / fp).exists():
+            break
+    with (open(MODEL_OUTPUT_DIR / source_dir / fp, "r", encoding="utf8") as i,
+          open(MODEL_OUTPUT_DIR / reval_dir / fp, "w", encoding="utf8") as o,
+          tqdm(total=3000, desc=fp) as pbar,
+          ):
+        for line in i:
+            res = json.loads(line)
+            if (res['game'].startswith(f"{game_filename(GAME_NAME)}")):
+                sid_prompt_dict = sid_prompt_dicts[res['game'].rsplit("_", 1)[-1]]
+                if (res['turn'] == 1):
+                    cur_sid = res["session"]
+                    prompt = sid_prompt_dict[cur_sid]
+                    cur_game = game_cls()
+                    cur_game.load_game(prompt)
+                    pbar.update(1)
+                elif solved == True:
+                    continue
+                else:
+                    assert cur_sid == res["session"]
+                solved, _ = cur_game.validate(res["response"])
+                if solved and not res["solved"]:
+                    count_pos += 1
+                elif not solved and res["solved"]:
+                    count_neg += 1
+                res["solved"] = solved
+            o.write(json.dumps(res))
+            o.write("\n")
+    return count_pos, count_neg
+
+
+if __name__ == "__main__":
+    def load(k):
+        with open(f"{PROBLEMSET_DIR}/{game_filename(GAME_NAME)}_{k}.json", "r", encoding="utf8") as f:
+            sid_prompt_dict = json.load(f)
+        return sid_prompt_dict
+    sid_prompt_dicts = {k: load(k) for k in map(str, range(1, 4))}
+    game_cls = _game_class_from_name(GAME_NAME)
+    for fp in OUTPUT_FILENAMES:
+        print(revalidate_bracket(fp))
+

reval_bracket_rerun.py

@@ -0,0 +1,46 @@
+# @title ##### Combine Rerun of the Bracket - All
+import os
+import json
+from tqdm import tqdm
+from pathlib import Path
+
+MODEL_OUTPUT_DIR = Path(os.getenv("TG_MODEL_OUTPUT_DIR", "model_outputs"))
+fd_new = MODEL_OUTPUT_DIR / "__runs__" / "_redo_bracket"
+fd_ori = MODEL_OUTPUT_DIR / "revalidate_anagram_3"
+fd_out = MODEL_OUTPUT_DIR / "revalidate_bracket_rerun"
+
+OUTPUT_FILENAMES = [
+    "results_gemma-2-9b-it.1s.jsonl",
+    "results_gemma-2-9b-it.zs.jsonl",
+    "results_gemma-2-27b-it.1s.jsonl",
+    "results_gemma-2-27b-it.zs.jsonl",
+
+    "results_llama-3.1-8b-instruct.1s.jsonl",
+    "results_llama-3.1-8b-instruct.zs.jsonl",
+    "results_llama-3.1-70b-instruct.1s.jsonl",
+    "results_llama-3.1-70b-instruct.zs.jsonl",
+    "results_llama-3.3-70b-instruct.1s.jsonl",
+    "results_llama-3.3-70b-instruct.zs.jsonl",
+
+    "results_qwen2-5-7b-instruct.1s.jsonl",
+    "results_qwen2-5-7b-instruct.zs.jsonl",
+    "results_qwen2-5-14b-instruct.1s.jsonl",
+    "results_qwen2-5-14b-instruct.zs.jsonl",
+    "results_qwen2-5-32b-instruct.1s.jsonl",
+    "results_qwen2-5-32b-instruct.zs.jsonl",
+    "results_qwen2-5-72b-instruct.1s.jsonl",
+    "results_qwen2-5-72b-instruct.zs.jsonl",
+]
+
+os.makedirs(fd_out, exist_ok=True)
+for fp in tqdm(OUTPUT_FILENAMES):
+    with open(fd_out / fp, "w", encoding="utf8") as o:
+        with open(fd_ori / fp, "r", encoding="utf8") as i:
+            for line in i:
+                res = json.loads(line)
+                if res['game'].startswith("Bracket Game"):
+                    continue
+                o.write(line)
+        with open((fd_new / fp).with_suffix(".6.jsonl"), "r", encoding="utf8") as i:
+            for line in i:
+                o.write(line)

reval_crosswords_all.py

@@ -0,0 +1,94 @@
+import os
+import json
+from tqdm import tqdm
+from textgames import GAME_NAMES, game_filename, _game_class_from_name
+from pathlib import Path
+
+GAME_NAME = GAME_NAMES[0]
+PROBLEMSET_DIR = Path(os.getenv("TG_PROBLEMSET_DIR", "problemsets"))
+MODEL_OUTPUT_DIR = Path(os.getenv("TG_MODEL_OUTPUT_DIR", "model_outputs"))
+OUTPUT_FILENAMES = [
+    # "results_gemma-2-9b-it.1s.jsonl",
+    # "results_gemma-2-9b-it.zs.jsonl",
+    # "results_gemma-2-27b-it.1s.jsonl",
+    # "results_gemma-2-27b-it.zs.jsonl",
+    #
+    # "results_llama-3.1-8b-instruct.1s.jsonl",
+    # "results_llama-3.1-8b-instruct.zs.jsonl",
+    # "results_llama-3.1-70b-instruct.1s.jsonl",
+    # "results_llama-3.1-70b-instruct.zs.jsonl",
+    # "results_llama-3.3-70b-instruct.1s.jsonl",
+    # "results_llama-3.3-70b-instruct.zs.jsonl",
+    #
+    # "results_qwen2-5-7b-instruct.1s.jsonl",
+    # "results_qwen2-5-7b-instruct.zs.jsonl",
+    # "results_qwen2-5-14b-instruct.1s.jsonl",
+    # "results_qwen2-5-14b-instruct.zs.jsonl",
+    # "results_qwen2-5-32b-instruct.1s.jsonl",
+    # "results_qwen2-5-32b-instruct.zs.jsonl",
+    # "results_qwen2-5-72b-instruct.1s.jsonl",
+    # "results_qwen2-5-72b-instruct.zs.jsonl",
+    #
+    # "results_deepseek-r1-distill-14b.1s.jsonl",
+    # "results_deepseek-r1-distill-14b.zs.jsonl",
+    # # "results_deepseek-r1-distill-14b.rerun.1s.jsonl",
+    #
+    # "results_chatgpt-4o-mini.1s.jsonl",
+    # "results_chatgpt-4o-mini.zs.jsonl",
+    # "results_chatgpt-o3-mini.zs.jsonl",
+    #
+    # # "results_qwen2-5-7b-instruct_sp.1s.jsonl",
+    # # "results_qwen2-5-7b-instruct_sp.zs.jsonl",
+
+    "results_deepseek-r1-distill-8b.1s.jsonl",
+    "results_deepseek-r1-distill-8b.zs.jsonl",
+]
+
+
+def revalidate_bracket(fp, reval_dir="revalidate_crosswords_all",
+                       source_dirs=("revalidate_bracket_all",)):
+    os.makedirs(MODEL_OUTPUT_DIR/reval_dir, exist_ok=True)
+    count_pos, count_neg = 0, 0
+    source_dir = "."
+    for source_dir in source_dirs:
+        if (MODEL_OUTPUT_DIR / source_dir / fp).exists():
+            break
+    with (open(MODEL_OUTPUT_DIR / source_dir / fp, "r", encoding="utf8") as i,
+          open(MODEL_OUTPUT_DIR / reval_dir / fp, "w", encoding="utf8") as o,
+          tqdm(total=3000, desc=fp) as pbar,
+          ):
+        for line in i:
+            res = json.loads(line)
+            if (res['game'].startswith(f"{game_filename(GAME_NAME)}")):
+                sid_prompt_dict = sid_prompt_dicts[res['game'].rsplit("_", 1)[-1]]
+                if (res['turn'] == 1):
+                    cur_sid = res["session"]
+                    prompt = sid_prompt_dict[cur_sid]
+                    cur_game = game_cls()
+                    cur_game.load_game(prompt)
+                    pbar.update(1)
+                elif solved == True:
+                    continue
+                else:
+                    assert cur_sid == res["session"]
+                solved, _ = cur_game.validate(res["response"])
+                if solved and not res["solved"]:
+                    count_pos += 1
+                elif not solved and res["solved"]:
+                    count_neg += 1
+                res["solved"] = solved
+            o.write(json.dumps(res))
+            o.write("\n")
+    return count_pos, count_neg
+
+
+if __name__ == "__main__":
+    def load(k):
+        with open(f"{PROBLEMSET_DIR}/{game_filename(GAME_NAME)}_{k}.json", "r", encoding="utf8") as f:
+            sid_prompt_dict = json.load(f)
+        return sid_prompt_dict
+    sid_prompt_dicts = {k: load(k) for k in map(str, range(1, 4))}
+    game_cls = _game_class_from_name(GAME_NAME)
+    for fp in OUTPUT_FILENAMES:
+        print(revalidate_bracket(fp))
+

reval_sudoku_all.py

@@ -0,0 +1,94 @@
+import os
+import json
+from tqdm import tqdm
+from textgames import GAME_NAMES, game_filename, _game_class_from_name
+from pathlib import Path
+
+GAME_NAME = GAME_NAMES[1]
+PROBLEMSET_DIR = Path(os.getenv("TG_PROBLEMSET_DIR", "problemsets"))
+MODEL_OUTPUT_DIR = Path(os.getenv("TG_MODEL_OUTPUT_DIR", "model_outputs"))
+OUTPUT_FILENAMES = [
+    # "results_gemma-2-9b-it.1s.jsonl",
+    # "results_gemma-2-9b-it.zs.jsonl",
+    # "results_gemma-2-27b-it.1s.jsonl",
+    # "results_gemma-2-27b-it.zs.jsonl",
+    #
+    # "results_llama-3.1-8b-instruct.1s.jsonl",
+    # "results_llama-3.1-8b-instruct.zs.jsonl",
+    # "results_llama-3.1-70b-instruct.1s.jsonl",
+    # "results_llama-3.1-70b-instruct.zs.jsonl",
+    # "results_llama-3.3-70b-instruct.1s.jsonl",
+    # "results_llama-3.3-70b-instruct.zs.jsonl",
+    #
+    # "results_qwen2-5-7b-instruct.1s.jsonl",
+    # "results_qwen2-5-7b-instruct.zs.jsonl",
+    # "results_qwen2-5-14b-instruct.1s.jsonl",
+    # "results_qwen2-5-14b-instruct.zs.jsonl",
+    # "results_qwen2-5-32b-instruct.1s.jsonl",
+    # "results_qwen2-5-32b-instruct.zs.jsonl",
+    # "results_qwen2-5-72b-instruct.1s.jsonl",
+    # "results_qwen2-5-72b-instruct.zs.jsonl",
+    #
+    # "results_deepseek-r1-distill-14b.1s.jsonl",
+    # "results_deepseek-r1-distill-14b.zs.jsonl",
+    # # "results_deepseek-r1-distill-14b.rerun.1s.jsonl",
+    #
+    # "results_chatgpt-4o-mini.1s.jsonl",
+    # "results_chatgpt-4o-mini.zs.jsonl",
+    # "results_chatgpt-o3-mini.zs.jsonl",
+    #
+    # # "results_qwen2-5-7b-instruct_sp.1s.jsonl",
+    # # "results_qwen2-5-7b-instruct_sp.zs.jsonl",
+
+    "results_deepseek-r1-distill-8b.1s.jsonl",
+    "results_deepseek-r1-distill-8b.zs.jsonl",
+]
+
+
+def revalidate_bracket(fp, reval_dir="revalidate_sudoku_all",
+                       source_dirs=("revalidate_crosswords_all",)):
+    os.makedirs(MODEL_OUTPUT_DIR/reval_dir, exist_ok=True)
+    count_pos, count_neg = 0, 0
+    source_dir = "."
+    for source_dir in source_dirs:
+        if (MODEL_OUTPUT_DIR / source_dir / fp).exists():
+            break
+    with (open(MODEL_OUTPUT_DIR / source_dir / fp, "r", encoding="utf8") as i,
+          open(MODEL_OUTPUT_DIR / reval_dir / fp, "w", encoding="utf8") as o,
+          tqdm(total=3000, desc=fp) as pbar,
+          ):
+        for line in i:
+            res = json.loads(line)
+            if (res['game'].startswith(f"{game_filename(GAME_NAME)}")):
+                sid_prompt_dict = sid_prompt_dicts[res['game'].rsplit("_", 1)[-1]]
+                if (res['turn'] == 1):
+                    cur_sid = res["session"]
+                    prompt = sid_prompt_dict[cur_sid]
+                    cur_game = game_cls()
+                    cur_game.load_game(prompt)
+                    pbar.update(1)
+                elif solved == True:
+                    continue
+                else:
+                    assert cur_sid == res["session"]
+                solved, _ = cur_game.validate(res["response"])
+                if solved and not res["solved"]:
+                    count_pos += 1
+                elif not solved and res["solved"]:
+                    count_neg += 1
+                res["solved"] = solved
+            o.write(json.dumps(res))
+            o.write("\n")
+    return count_pos, count_neg
+
+
+if __name__ == "__main__":
+    def load(k):
+        with open(f"{PROBLEMSET_DIR}/{game_filename(GAME_NAME)}_{k}.json", "r", encoding="utf8") as f:
+            sid_prompt_dict = json.load(f)
+        return sid_prompt_dict
+    sid_prompt_dicts = {k: load(k) for k in map(str, range(1, 4))}
+    game_cls = _game_class_from_name(GAME_NAME)
+    for fp in OUTPUT_FILENAMES:
+        print(revalidate_bracket(fp))
+

textgames/__init__.py

@@ -14,8 +14,10 @@ from pandas import read_csv
 import json
 
 
-# ["🔑\tPassword Game", "🧩\tText Sudoku", "🗳️\tBracket Game", "📈\tOrdering Text",
-#  "🏝️\tIslands", "🔎\tString Search", "📰\tCrossword Arranger", "🔤\tAnagram Scribble",]
+# [
+#  "📰\tCrossword Arranger", "🧩\tText Sudoku", "🏝️\tIslands", "🔑\tPassword Game",
+#  "📈\tOrdering Text", "🔤\tAnagram Scribble", "🗳️\tBracket Game", "🔎\tString Search",
+#  ]
 THE_GAMES = {
     k: v.get_game_name() for k, v in [
         ("1", CrosswordArrangerGame),
@@ -60,12 +62,13 @@ def _game_class_from_name(game_name):
     return None
 
 
-def preload_game(game_name, level_id, user):
+def preload_game(game_name, level_id, user, sid=None):
     game_cls = _game_class_from_name(game_name)
-    email_sid_dict = read_csv(
-        f"{os.getenv('TEXTGAMES_OUTPUT_DIR')}/textgames_userauth.tsv", sep='\t'
-    ).dropna().set_index("EMAIL").SID.to_dict()
-    sid = email_sid_dict.get(user["email"])
+    if not sid:
+        email_sid_dict = read_csv(
+            f"{os.getenv('TEXTGAMES_OUTPUT_DIR')}/textgames_userauth.tsv", sep='\t'
+        ).dropna().set_index("EMAIL").SID.to_dict()
+        sid = email_sid_dict.get(user["email"])
     print(f"preload_game('{game_name}', '{level_id}', '{user['email']}') on {sid}")
 
     with open(f"problemsets/{game_filename(game_name)}_{level_id}.json", "r", encoding="utf8") as f:

textgames/anagram_scribble/anagram_scribble.py

@@ -5,6 +5,7 @@ import json
 import string
 import re
 
+
 class AnagramScribble(BaseGame):
     @staticmethod
     def get_game_name() -> str:
@@ -43,6 +44,18 @@ class AnagramScribble(BaseGame):
         if total_chars_extraction != "Error loading game state.":
             characters = total_chars_extraction.split(",")
             self.total_chars = [char.strip().strip("'") for char in characters]
+        self.possible_ans = ""
+        _chars = sorted(self.total_chars)
+        for w in self.WORD_LIST_BIN[str(self.num_chars)]:
+            _ans = sorted(w)
+            j, k = 0, 0
+            while j < len(_ans) and k < len(_chars):
+                if _ans[j] == _chars[k]:
+                    j += 1
+                k += 1
+            if j >= len(_ans):
+                self.possible_ans = w
+                break
 
     def _generate_new_game(self, *args, **kwargs) -> None:
         self.low_num_chars = kwargs['low_num_chars']
@@ -57,16 +70,16 @@ class AnagramScribble(BaseGame):
 
     def _get_prompt(self) -> str:
         if self.allow_repeat:
-            prompt = f"Construct a valid {self.num_chars}-character English word from the following letters:\n{self.total_chars}.\nEach character can be used multiple times. Please write None if there is no valid combination."
+            prompt = f"Construct a valid {self.num_chars}-character English word from the following letters:\n{self.total_chars}.\nEach character can be used multiple times. Please write None if there is no valid combination. Print only the answer.\n"
         else:
-            prompt = f"Construct a valid {self.num_chars}-character English word from the following letters:\n{self.total_chars}.\nEach character can only be used once. Please write None if there is no valid combination."
+            prompt = f"Construct a valid {self.num_chars}-character English word from the following letters:\n{self.total_chars}.\nEach character can only be used once. Please write None if there is no valid combination. Print only the answer.\n"
         return prompt
     
     def _validate(self, answer: str) -> (bool, str):
-        answer = answer.lower()
-        if self.possible_ans != "" and answer == "none":
+        if self.possible_ans != "" and answer == "None":
             val_msg = "There is a valid answer."
             return False, val_msg
+        answer = answer.lower()
         if len(answer) != self.num_chars:
             val_msg = f"Your answer must be exactly {self.num_chars} characters long"
             return False, val_msg
@@ -74,12 +87,31 @@ class AnagramScribble(BaseGame):
             if char not in self.total_chars:
                 val_msg = "Your answer must only contain the characters provided"
                 return False, val_msg
-        if (not self.allow_repeat and (len(set(answer)) != len(answer))
-                and (len(self.possible_ans) == len(set(self.possible_ans)))):
-            val_msg = "Your answer must not contain repeated characters"
-            return False, val_msg
+        # if (not self.allow_repeat and (len(set(answer)) != len(answer))
+        #         and (len(self.possible_ans) == len(set(self.possible_ans)))):
+        if not self.allow_repeat:
+            _ans = sorted(answer)
+            _chars = sorted(self.total_chars)
+            j, k = 0, 0
+            while j < len(_ans) and k < len(_chars):
+                if _ans[j] == _chars[k]:
+                    j += 1
+                k += 1
+            if j < len(_ans):
+                val_msg = "Your answer must not contain repeated characters"
+                return False, val_msg
         if answer not in self.WORD_LIST_BIN[str(self.num_chars)]:
             val_msg = "Your answer is not a valid English word"
             return False, val_msg
 
         return True, ""
+
+    @staticmethod
+    def example() -> (str, str):
+        prompt = ("Construct a valid 5-character English word from the following letters:\n"
+                  "['e', 'l', 'o', 'b', 's', 'p'].\n"
+                  "Each character can be used multiple times. Please write None if there is no valid combination."
+                  " Print only the answer.\n")
+        answer = "sleep"
+        return prompt, answer
+

textgames/bracket_game/bracket_game.py

@@ -1,5 +1,6 @@
 import random
 import re
+from bisect import bisect_left
 from pathlib import Path
 from textgames.base_game import BaseGame
 #%%
@@ -57,48 +58,94 @@ class BracketGame(BaseGame):
                 self.MULTI_WORD_LIST.append(self.WORD_LIST[num1] + self.WORD_LIST[num2])
 
     def _validate(self, answer: str) -> (bool, str):
-        for rule in self.rules:
-            arr = answer.split(rule[0])
-            
-            if rule[1][1] not in arr[0] or rule[1][2] not in arr[1]:
-                val_msg = f"{rule[0]} is not between the correct bracket, {rule[1][1]} not in {arr[0]} and {rule[1][2]} not in {arr[1]}"
-                return False, val_msg
-            
-        filter_answer = answer
-        for i in range(0, 26):
-            cc = chr(ord("a") + i)
-            filter_answer = filter_answer.replace(cc,"")
+        answer = "".join(answer.split()).lower()
 
-            cc = chr(ord("A") + i)
-            filter_answer = filter_answer.replace(cc,"")
-        
-        open_bracket_list = ["[", "{", "(", "<"]
-        close_bracket_map = {
-            "[":"]", "{":"}", "(":")", "<":">"
-        }
-
-        # check max depth
-        count = 0
-        st = []
-
-        for i in range(len(filter_answer)):
-            if (filter_answer[i] in open_bracket_list):
-                st.append(filter_answer[i]) # pushing the bracket in the stack
-            else:
-                if len(st) > 0 and (filter_answer[i] == close_bracket_map[st[-1]]):
-                    if (count < len(st)):
-                        count = len(st)
-                    st.pop()
-                else:
+        if ("".join(filter(lambda a: a.isalpha(), answer)) !=
+                "".join(filter(lambda a: a.isalpha(), self.string.lower()))):
+            val_msg = f"You are not allowed to change the character sequence of base text '{self.string}'."
+            return False, val_msg
+
+        char2type_op = {b[1]: b[0] for b in self.BRACKETS}
+        char2type_ed = {b[2]: b[0] for b in self.BRACKETS}
+
+        depth_count = {b[0]: [(-1, 0)] for b in self.BRACKETS}
+
+        def push(dc, v):
+            cur_depth = dc[-1][-1]
+            if cur_depth < 0:
+                return False
+            dc.append((i, cur_depth + v))
+            return True
+
+        mak, cur_mak = 0, 0
+        for i, c in enumerate(answer):
+            if c in char2type_op:
+                push(depth_count[char2type_op[c]], 1)
+                cur_mak += 1
+            elif c in char2type_ed:
+                if not push(depth_count[char2type_ed[c]], -1):
                     val_msg = "There is a closing bracket without an open bracket"
                     return False, val_msg
-        
-        if count == self.depth:
-            return True, ""
-        else:       
-            val_msg = f"The depth of the bracket is {count}. The expected depth is {self.depth}"
+                cur_mak -= 1
+            mak = max(mak, cur_mak)
+
+        if mak != self.depth:
+            val_msg = f"The depth of the bracket is {mak}. The expected depth is {self.depth}"
             return False, val_msg
 
+        for rule in self.rules:
+            i = answer.find(rule[0])
+            if i < 0:
+                val_msg = f"The text '{rule[0]}' is not found in your answer."
+                return False, val_msg
+
+            i_depth = bisect_left(depth_count[rule[1][0]], (i, -1)) - 1
+            if depth_count[rule[1][0]][i_depth][-1] < 1:
+                val_msg = f"The text '{rule[0]}' is not inside any {rule[1][0]} bracket {rule[1][1]} {rule[1][2]}"
+                return False, val_msg
+
+            # arr = answer.split(rule[0])
+            # if rule[1][1] not in arr[0] or rule[1][2] not in arr[1]:
+            #     val_msg = f"The text '{rule[0]}' is not between the correct bracket, {rule[1][1]} not in {arr[0]} and {rule[1][2]} not in {arr[1]}"
+            #     return False, val_msg
+
+        return True, ""
+
+        # filter_answer = answer
+        # for i in range(0, 26):
+        #     cc = chr(ord("a") + i)
+        #     filter_answer = filter_answer.replace(cc,"")
+        #
+        #     cc = chr(ord("A") + i)
+        #     filter_answer = filter_answer.replace(cc,"")
+        #
+        # open_bracket_list = ["[", "{", "(", "<"]
+        # close_bracket_map = {
+        #     "[":"]", "{":"}", "(":")", "<":">"
+        # }
+        #
+        # # check max depth
+        # count = 0
+        # st = []
+        #
+        # for i in range(len(filter_answer)):
+        #     if (filter_answer[i] in open_bracket_list):
+        #         st.append(filter_answer[i]) # pushing the bracket in the stack
+        #     else:
+        #         if len(st) > 0 and (filter_answer[i] == close_bracket_map[st[-1]]):
+        #             if (count < len(st)):
+        #                 count = len(st)
+        #             st.pop()
+        #         else:
+        #             val_msg = "There is a closing bracket without an open bracket"
+        #             return False, val_msg
+        #
+        # if count == self.depth:
+        #     return True, ""
+        # else:
+        #     val_msg = f"The depth of the bracket is {count}. The expected depth is {self.depth}"
+        #     return False, val_msg
+
     def _generate_new_game(self, *args, **kwargs) -> None:
         num_words = kwargs["num_words"]
         num_rules = kwargs["num_rules"]
@@ -141,6 +188,7 @@ class BracketGame(BaseGame):
         prompt = f"You are given a text {self.string} Your job is to put some valid parenthesis brackets in the text such that:\n"
         for rule in self.rules:
             prompt += f"- \"{rule[0]}\" is inside a {rule[1][0]} bracket\n"
+        prompt += "The open and close parenthesis for block is [ ], curly is { }, round is ( ), and angle is < >\n"
         prompt += f"The bracket depth must be {self.depth} and print only the answer\n"
         return prompt
 
@@ -159,7 +207,7 @@ class BracketGame(BaseGame):
             else:
                 return 0
             
-        content = state_string.split("the text such that:")[1].split("\nThe bracket depth must be")[0].split("\n")
+        content = state_string.split("the text such that:")[1].split("\nThe open and close parenthesis ")[0].split("\n")
 
         self.words = []
         self.rules = []
@@ -188,3 +236,14 @@ class BracketGame(BaseGame):
         self.create_multiple_words()        
 
         sort_game_states(self)
+
+    @staticmethod
+    def example() -> (str, str):
+        prompt = ("You are given a text fabuloustextgames Your job is to put some valid parenthesis brackets in the text such that:\n"
+                  "- \"games\" is inside a round bracket\n"
+                  "- \"text\" is inside a angle bracket\n"
+                  "- \"fabulous\" is inside a block bracket\n"
+                  "The open and close parenthesis for block is [ ], curly is { }, round is ( ), and angle is < >\n"
+                  "The bracket depth must be 2 and print only the answer\n")
+        answer = "[[fabulous]<text>(games)]"
+        return prompt, answer

textgames/crossword_arranger/crossword_arranger.py

@@ -125,19 +125,47 @@ class CrosswordArrangerGame(BaseGame):
         return prompt
 
     def _validate(self, answer: str) -> (bool, str):
-        ans_hor = list(filter(None, answer.lower().replace(' ', '\n').split("\n")))
+        answer = answer if answer else ""
+        # ans_hor = list(filter(None, answer.lower().replace(' ', '\n').split("\n")))
+        ans_hor = answer.lower().split()
         val_msg = ""
+        if len(ans_hor) != self.board_size:
+            arr = answer.lower().split()
+            if all(len(l) == 1 for l in arr) and (len(arr) == self.board_size * self.board_size):
+                ans_hor = ["".join(arr[i:i+self.board_size]) for i in range(0, len(arr), self.board_size)]
         if len(ans_hor) != self.board_size:
             val_msg = f"Mismatch answer length found!! Expected size of {self.board_size}, got {len(ans_hor)}."
             return False, val_msg
+        for w in ans_hor:
+            if len(w) != self.board_size:
+                val_msg = f"Mismatch answer length found!! Expected size of {self.board_size}, got {len(w)}."
+                return False, val_msg
         ans_ver = [''.join(ans_hor[r][c] for r in range(self.board_size)) for c in range(self.board_size)]
         word_set = set(self.word_list)
-        for w in chain(ans_hor, ans_ver):
+        for i, w in enumerate(chain(ans_hor, ans_ver)):
             if w not in word_set:
+                val_msg = (f"Mismatch answer word found!! {'Horizontal' if i < self.board_size else 'Vertical'} word"
+                           f" '{w}' is not in the word set.")
                 return False, val_msg
             word_set.remove(w)
         return True, val_msg
 
+    @staticmethod
+    def example() -> (str, str):
+        prompt = (f"Given a board size of 3x3, arrange a possible crossword puzzle answer from a list of words.\n"
+                  f"Item in the list can only be used once.\n\n"
+                  f"List of words:\n"
+                  f"- app\n"
+                  f"- all\n"
+                  f"- and\n"
+                  f"- lee\n"
+                  f"- let\n"
+                  f"- pat\n"
+                  f"- pee\n"
+                  f"- pet\n\n"
+                  f"Print only the answer.")
+        answer = "app\nlee\nlet"
+        return prompt, answer
 
 #%%
 

textgames/islands/islands.py

@@ -99,8 +99,8 @@ class Islands(BaseGame):
         answer = [a.replace(" ", "").lower().strip() for a in answer]
 
         # check the size
-        if len(answer) != self.N or len(answer[0]) != self.N:
-            val_msg = f"2D grid is not {self.N} x {self.N}. ({len(answer)} x {len(answer[0])})"
+        if len(answer) != self.N or any((len(a) < self.N) for a in answer):
+            val_msg = f"2D grid is not {self.N} x {self.N}. ({len(answer)} x {set(len(a) for a in answer)})"
             return False, val_msg
 
         # check the tiles, ensure they are valid
@@ -194,4 +194,16 @@ Your 2D grid must follow the following rules:
 
 Print only the answer.
 """
-        return prompt
+        return prompt
+
+    @staticmethod
+    def example() -> (str, str):
+        prompt = ("You are asked to construct a 2D 5 x 5 grid, consisting of water tiles (denoted by \u2019.\u2019), \n"
+                  "land tiles (denoted by \u2019#\u2019). \n\n"
+                  "A group of connected land tiles in 4 cardinal directions forms an island.\n\n"
+                  "Your 2D grid must follow the following rules:\n"
+                  "- There must be exactly 1 islands.\n"
+                  "- The size of each island must be from 1 to 2 tiles.\n\n"
+                  "Print only the answer.\n")
+        answer = "...##\n.....\n.....\n.....\n....."
+        return prompt, answer

textgames/ordering_text/ordering_text.py

@@ -5,10 +5,10 @@ Rules Description
 
 word length:
 - example: word less than 5 characters gets 10 points
-- possible operands: {\eq, \lt, \gt, \ne}
-    - \le and \ge will be randomized for prompt generation
-- possible combinations: {\gt\lt, \gt\lt\ne}
-- only 1 \ne is considered
+- possible operands: {\\eq, \\lt, \\gt, \\ne}
+    - \\le and \\ge will be randomized for prompt generation
+- possible combinations: {\\gt\\lt, \\gt\\lt\\ne}
+- only 1 \\ne is considered
 
 neighboring / consecutive chars
 - example: every pair of consecutive consonant gets 5 points
@@ -66,6 +66,15 @@ from textgames.base_game import BaseGame
 from textgames.assets.word_list import WORDS_LIST, WORDS_BY_LEN
 
 
+#%%
+index_to_word = {
+    1: "first", 2: "second", 3: "third", 4: "fourth", 5: "fifth",
+    6: "sixth", 7: "seventh", 8: "eighth", 9: "ninth", 10: "tenth",
+    11: "eleventh", 12: "twelfth", 13: "thirteenth", 14: "fourteenth",
+    15: "fifteenth", 16: "sixteenth", 17: "seventeenth", 18: "eighteenth",
+}
+
+
 #%%
 class Scoring:
     def __init__(self, point: int):
@@ -505,14 +514,15 @@ class OrderingTextGame(BaseGame):
         return self.answer    # sorted(self.words, key=lambda word: (self.get_point(word), word))
 
     def _validate(self, answer: str) -> (bool, str):
-        answer = answer.lower().replace(' ', '\n')
-        if answer != "\n".join(self.get_answer()):
-            for i, (a, b) in enumerate(zip(answer.split(), self.get_answer()), 1):
-                if a != b:
-                    val_msg = f"{a} is not supposed to be at index {i}."
-                    return False, val_msg
-        else:
-            return True, ""
+        answer = answer.lower().replace(',', ' ').split()
+        gold = self.get_answer()
+        if len(answer) < len(gold):
+            return False, f"Your answer is too short. There should be {len(gold)} items."
+        for i, (a, b) in enumerate(zip(answer, self.get_answer()), 1):
+            if a != b:
+                val_msg = f"'{a}' is not supposed to be the {index_to_word[i]} word in the order."
+                return False, val_msg
+        return True, ""
 
     def _generate_new_game(self, *args, **kwargs) -> None:
         if "preset_config" in kwargs:
@@ -588,6 +598,26 @@ class OrderingTextGame(BaseGame):
         prompt += "\nPrint only the answer."
         return prompt
 
+    @staticmethod
+    def example() -> (str, str):
+        prompt = ("Given a set of rules to calculate point, sort the set of words in decreasing order.\n"
+                  "When there 2 or more words with same point, sort lexicographically.\n\n"
+                  "Rules:\n"
+                  "- add 10 points if there exists 'u' in the word\n\n"
+                  "Words:\n"
+                  "- hudi\n"
+                  "- genta\n"
+                  "- aji\n"
+                  "- ruochen\n\n"
+                  "Print only the answer.")
+        answer = (
+            "hudi\n"
+            "ruochen\n"
+            "aji\n"
+            "genta"
+        )
+        return prompt, answer
+
 
 #%%
 

textgames/password_game/password_game.py

@@ -274,3 +274,13 @@ class PasswordGame(BaseGame):
         self.rules = [rule for rule in new_rules]
 
         sort_game_states(self)
+
+    @staticmethod
+    def example() -> (str, str):
+        prompt = ("Please write a text string without any space by following a set of given rules."
+                  " Please write only the answer and follow the following criteria:\n"
+                  "- the text has 6 english character\n"
+                  "- the text has 0 uppercase characters\n")
+        answer = "hoodie"
+        return prompt, answer
+

textgames/string_search/string_search.py

@@ -309,4 +309,16 @@ Find a substring of exactly {self.answer_len} characters long that:
 {extra_constraints}
 Print only the answer.
 """
-        return prompt
+        return prompt
+
+    @staticmethod
+    def example() -> (str, str):
+        prompt = ("You are given the following string:\n"
+                  "hudigentaajiruochen\n\n"
+                  "Find a substring of exactly 3 characters long that:\n"
+                  " - Contains t\n"
+                  " - Does not contain i and a\n\n"
+                  "Print only the answer.\n")
+        answer = "ent"
+        return prompt, answer
+

textgames/sudoku/sudoku.py

@@ -9,6 +9,7 @@ Please solve the 9x9 sudoku with 1,2,3,4,5,6,7,8,9 as the values and fill _ with
 Print only the answer.
 """
 
+
 #%%
 class Sudoku(BaseGame):
     @staticmethod
@@ -28,34 +29,47 @@ class Sudoku(BaseGame):
             for j in range(self.size):
                 num = mat[i][j]
                 if num == self.empty_character:
-                    return False
+                    return False, "There are unfilled cells"
     
                 subgrid_index = (i // self.srn) * self.srn + (j // self.srn)
     
-                if num in rows[i] or num in cols[j] or num in subgrids[subgrid_index]:
-                    return False
-    
+                if num in rows[i]:
+                    return False, f"Duplicated row value ({num}) for cell in row {i+1} column {j+1}."
+                elif num in cols[j]:
+                    return False, f"Duplicated column value ({num}) for cell in row {i+1} column {j+1}."
+                elif num in subgrids[subgrid_index]:
+                    return False, f"Duplicated subgrid value ({num}) for cell in row {i+1} column {j+1}."
+
                 rows[i].add(num)
                 cols[j].add(num)
                 subgrids[subgrid_index].add(num)
 
-        return True
+        return True, ""
 
     def _validate(self, input) -> (bool, str):
         mat = [[self.empty_character for i in range(self.size)] for j in range(self.size)]
 
+        input = input if input else ""
         arr = input.split()
+        if all(len(l) == 1 for l in arr) and (len(arr) == self.size * self.size):
+            arr = ["".join(arr[i:i+self.size]) for i in range(0, len(arr), self.size)]
+        if (len(arr) != self.size) or any(len(arr[i]) != self.size for i in range(len(arr))):
+            arr = input.split("\n")
+            val_msg = f"Your answer is wrong in shape, it should be {self.size}x{self.size} sudoku."
+            return False, val_msg
+
         for i in range(len(arr)):
             for j in range(len(arr[i])):
                 if arr[i][j] not in self.char_to_id:
-                    val_msg = "Found unrecognized character(s)"
+                    val_msg = "There are unrecognized characters, or possibly unfilled cells."
                     return False, val_msg
                 
                 mat[i][j] = self.char_to_id[arr[i][j]]
                 if arr[i][j] != self.mat[i][j] and self.mat[i][j] != self.empty_character:
                     val_msg = "One or more characters are replaced"
                     return False, val_msg
-        return self.is_valid_sudoku(mat), ""
+
+        return self.is_valid_sudoku(mat)
 
     def _generate_new_game(self, *args, **kwargs) -> None:
         size=kwargs["size"]
@@ -228,4 +242,14 @@ class Sudoku(BaseGame):
         self.char_to_id = {}
         for c_id in range(len(self.characters)):
             self.char_to_id[self.characters[c_id]] = c_id
-        
+
+    @staticmethod
+    def example() -> (str, str):
+        prompt = ("Please solve the 4x4 sudoku with A,B,C,D as the values and fill _ with the possible value and"
+                  " only print the answer. Follow the sudoku rule.\nA_CD CD_B _AD_ DCBA")
+        answer = ("ABCD\n"
+                  "CDAB\n"
+                  "BADC\n"
+                  "DCBA")
+        return prompt, answer
+