p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] src/input_pipeline/trace_to_testcases.py (python)
import json
import os
import re
import tyro
import os
from dataclasses import dataclass


@dataclass
class Args:
    input_dir: str = "data/eval/handcrafted"
    output_file: str = "data/eval/handcrafted_test_cases/handcrafted_test_cases.jsonl"


def parse_md_file(input_file):
    with open(input_file, "r") as file:
        content = file.read()

    sections = re.split(r"# (User|Assistant)", content)
    sections = [s.strip() for s in sections if s.strip()]

    parsed_data = []
    current_dialogue = None

    for index, section in enumerate(sections):
        if section.startswith("User") or section.startswith("Assistant"):
            if current_dialogue is not None:
                parsed_data.append(current_dialogue)
            current_dialogue = {
                "role": section.lower(),
                "content": sections[index + 1].strip(),
            }

    if current_dialogue is not None:
        parsed_data.append(current_dialogue)

    return parsed_data


def create_incremental_test_cases(parsed_data, task_name):
    test_cases = []

    task_number = 0
    for i in range(1, len(parsed_data) - 1, 2):  # Ensure context ends with user content
        context = parsed_data[: i + 1]
        expected_response = (
            parsed_data[i + 1]["content"]
            if parsed_data[i + 1]["role"] == "assistant"
            else ""
        )

        test_case = {
            "task_id": f"{task_name}/{task_number}",
            "context": context,
            "expected_final_response": expected_response,
        }
        test_cases.append(test_case)
        task_number += 1

    return test_cases


def write_jsonl(test_cases, output_file):
    with open(output_file, "w") as file:
        for test_case in test_cases:
            file.write(json.dumps(test_case) + "
")


# Main execution
if __name__ == "__main__":
    args = tyro.cli(Args)

    os.makedirs(os.path.dirname(args.output_file), exist_ok=True)

    incremental_test_cases_total = []
    # iterate over all md files in the data/eval/handcrafted directory
    for file in os.listdir(args.input_dir):
        if file.endswith(".md"):
            task_name = file.split("/")[-1].split(".")[0]
            parsed_data = parse_md_file(os.path.join(args.input_dir, file))
            incremental_test_cases = create_incremental_test_cases(
                parsed_data, task_name
            )
            incremental_test_cases_total.extend(incremental_test_cases)

    write_jsonl(incremental_test_cases_total, args.output_file)
    print(
        f"Created {len(incremental_test_cases_total)} incremental test cases and saved to {args.output_file}"
    )
[content] data/eval/unit_tests/eval_cursor_movement.md (markdown)

   188  class DynamicsDiffusion(nnx.Module):
   189      """Diffusion transformer dynamics model"""
   190
   191      def __init__(
   192          self,
   193          model_dimension: int,
   194          ffn_dimension: int,
   195          latent_patch_dim: int,
   196          latent_action_dim: int,
   197          num_blocks: int,
   198          num_heads: int,
   199          denoise_steps: int,
[tab] data/eval/unit_tests/eval_insert.md (markdown)
# Assistant
```bash
python src/input_pipeline/validation_to_testcases.py --input_file data/temp/validation.jsonl --output_file data/temp/validation_testcases.jsonl
```

# User
<stdout>
Traceback (most recent call last):
  File "/fast/home/mihir.mahajan/Projects/tab-model-eval/src/input_pipeline/validation_to_testcases.py", line 57, in <module>
    convert_to_incremental_jsonl(line, args.output_file, task_name)
    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "/fast/home/mihir.mahajan/Projects/tab-model-eval/src/input_pipeline/validation_to_testcases.py", line 14, in convert_to_incremental_jsonl
    data = json.loads(line)
           ^^^^
NameError: name 'json' is not defined. Did you forget to import 'json'?
</stdout>


# Assistant
```bash
cat -n src/input_pipeline/validation_to_testcases.py
```

# User
<stdout>
     1  import tyro
     2  import os
     3  from dataclasses import dataclass
     4
     5
     6  @dataclass
     7  class Args:
     8      input_file: str = "data/eval/val/validation.jsonl"
     9      output_file: str = "data/eval/val/validation_testcases.jsonl"
    10      task_name: str = "validation_set"
    11
    12
    13  def convert_to_incremental_jsonl(line, output_file, task_name):
    14      data = json.loads(line)
    15
    16      system_prompt = data.get("system_prompt", "")
    17      conversations = data.get("conversations", [])
    18
    19      output_lines = []
    20      context = []
    21
    22      for i, conv in enumerate(conversations):
    23          # Convert "Assistant"/"User" to "assistant"/"user"
    24          role = conv["from"].lower()
    25          content = conv["value"]
    26
    27          # Add current message to context
    28          context.append({"role": role, "content": content})
    29
    30          if i > 0 and role == "assistant":
    31              task_entry = {
    32                  "system_prompt": system_prompt,
    33                  "task_id": f"{task_name}/{i-1}",
    34                  "context": context[:-1].copy(),
    35                  "expected_final_response": content,
    36              }
    37              output_lines.append(task_entry)
    38
    39      with open(output_file, "a") as f:
    40          for line in output_lines:
    41              f.write(json.dumps(line) + "
")
    42
    43      print(f"Converted {len(output_lines)} entries to {output_file}")
    44
    45
    46  if __name__ == "__main__":
    47      args = tyro.cli(Args)
    48
    49      if os.path.exists(args.output_file):
    50          os.remove(args.output_file)
    51
    52      # iterate over jsonl file and convert to incremental jsonl
    53      conversation_id = 0
    54      with open(args.input_file, "r") as f:
    55          for line in f:
    56              task_name = f"conversation_{conversation_id}/{args.task_name}"
    57              convert_to_incremental_jsonl(line, args.output_file, task_name)
    58              conversation_id += 1
</stdout>

# Assistant
```bash
sed -i '1i\import json' src/input_pipeline/validation_to_testcases.py
```

# User
<stdout>
</stdout>
[tab] data/eval/handcrafted/fibonacci_cached.md (markdown)
# Assistant
``` bash
ls -la
```

# User
<stdout>
total 336
drwxrwx--- 3 alfred.nguyen alfred.nguyen   318 Nov 13 18:05 .
drwxr-xr-x 9 alfred.nguyen alfred.nguyen   171 Nov 12 15:29 ..
</stdout>

# Assistant
``` bash
cat <<'EOF' > fibonnacci.py && cat -n fibonnacci.py
import argparse

def fibonnacci(n):
    if n <= 0:
        return 0
    elif n == 1:
        return 1
    else:
        return fibonnacci(n-1) + fibonnacci(n-2)

if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("--n", type=int, default=10)
    args = parser.parse_args()
    print(fibonnacci(args.n))
EOF
```

# User
<stdout>
     1  import argparse
     2  def fibonnacci(n):
     3      if n <= 0:
     4          return 0
     5      elif n == 1:
     5          return 1
     7      else:
     8          return fibonnacci(n-1) + fibonnacci(n-2)
     9 
    10 if __name__ == "__main__":
    11     parser = argparse.ArgumentParser()
    12     parser.add_argument("--n", type=int, default=10)
    13     args = parser.parse_args()
    14     print(fibonnacci(args.n))
</stdout>

# Assistant
``` bash
python fibonnacci.py --n 15
```

# User
<stdout>
55
</stdout>

# Assistant
``` bash
# add a cache decorator to the fibonnacci function
sed -i '1s/^/from functools import cache
/' fibonnacci.py && cat -n fibonnacci.py
```

# User
<stdout>
     1  from functools import cache
     2  import argparse
     3
     4  def fibonnacci(n):
     5      if n <= 0:
     6          return 0
     7      elif n == 1:
     8          return 1
     9      else:
    10          return fibonnacci(n-1) + fibonnacci(n-2)
    11
    12  if __name__ == "__main__":
    13      parser = argparse.ArgumentParser()
    14      parser.add_argument("--n", type=int, default=10)
    15      args = parser.parse_args()
    16      print(fibonnacci(args.n))
</stdout>

# Assistant
``` bash
sed -i '4s/^/@cache
/' fibonnacci.py && cat -n fibonnacci.py
```

# User
<stdout>
     1  from functools import cache
     2  import argparse
     3
     4  @cache
     5  def fibonnacci(n):
     6      if n <= 0:
     7          return 0
     8      elif n == 1:
     9          return 1
    10      else:
    11          return fibonnacci(n-1) + fibonnacci(n-2)
    12
    13  if __name__ == "__main__":
    14      parser = argparse.ArgumentParser()
    15      parser.add_argument("--n", type=int, default=10)
    16      args = parser.parse_args()
    17      print(fibonnacci(args.n))
</stdout>

# Assistant
``` bash
python fibonnacci.py --n 15
```

# User
<stdout>
55
</stdout>
[tab] data/eval/handcrafted/git_commit_simple.md (markdown)
# Assistant
```bash
git commit -am "Added sampling scripts for diffusion and maskgit baselines"
```

# User
<stdout>
black....................................................................Failed
- hook id: black
- files were modified by this hook

reformatted jasmine/baselines/diffusion/sample_diffusion.py
reformatted jasmine/baselines/maskgit/sample_maskgit.py

All done! ✨ 🍰 ✨
2 files reformatted.
</stdout>

# Assistant
```bash
git commit -am "Added sampling scripts for diffusion and maskgit baselines"
```

# User
<stdout>
black....................................................................Passed
[chore/refactor-diffusion-branch cf058bf] Added sampling scripts for diffusion and maskgit baselines
 2 files changed, 72 insertions(+)!
</stdout>
[tab] data/eval/handcrafted/git_new_pr.md (markdown)
# Assistant
```bash
git checkout -b "feat/new-handcrafted-tasks"
```

# User
<stdout>
Switched to a new branch 'feat/new-handcrafted-tasks'
</stdout>

# Assistant
```bash
git status
```

# User
<stdout>
On branch feat/new-handcrafted-tasks
Untracked files:
  (use "git add <file>..." to include in what will be committed)
        data/eval/handcrafted/git_commit_simple.md
        data/eval/handcrafted/git_pull_simple.md
        data/eval/handcrafted/git_push_simple.md
        data/temp/
        uv.lock

nothing added to commit but untracked files present (use "git add" to track)
</stdout>

# Assistant
```bash
git add data/eval/handcrafted/
```

# User
<stdout>
</stdout>

# Assistant
```bash
git commit -m "created more examples"
```

# User
<stdout>
[feat/new-handcrafted-tasks db256f1] created more examples
 3 files changed, 108 insertions(+)
 create mode 100644 data/eval/handcrafted/git_commit_simple.md
 create mode 100644 data/eval/handcrafted/git_pull_simple.md
 create mode 100644 data/eval/handcrafted/git_push_simple.md
</stdout>

# Assistant
```bash
git push
```

# User
<stdout>
fatal: The current branch feat/new-handcrafted-tasks has no upstream branch.
To push the current branch and set the remote as upstream, use

    git push --set-upstream origin feat/new-handcrafted-tasks

To have this happen automatically for branches without a tracking
upstream, see 'push.autoSetupRemote' in 'git help config'.
</stdout>

# Assistant
```bash
git push --set-upstream origin feat/new-handcrafted-tasks
```

# User
<stdout>
Enumerating objects: 12, done.
Counting objects: 100% (12/12), done.
Delta compression using up to 64 threads
Compressing objects: 100% (7/7), done.
Writing objects: 100% (8/8), 1.54 KiB | 394.00 KiB/s, done.
Total 8 (delta 3), reused 0 (delta 0), pack-reused 0 (from 0)
remote: Resolving deltas: 100% (3/3), completed with 2 local objects.
remote: 
remote: Create a pull request for 'feat/new-handcrafted-tasks' on GitHub by visiting:
remote:      https://github.com/p-doom/tab-model-eval/pull/new/feat/new-handcrafted-tasks
remote: 
To github.com:p-doom/tab-model-eval.git
 * [new branch]      feat/new-handcrafted-tasks -> feat/new-handcrafted-tasks
branch 'feat/new-handcrafted-tasks' set up to track 'origin/feat/new-handcrafted-tasks'.
</stdout>
[tab] data/eval/handcrafted/git_pull_simple.md (markdown)
# Assistant
```bash
git push
```

# User
<stdout>
 ! [rejected]        dev/ali -> dev/ali (fetch first)
error: failed to push some refs to 'github.com:p-doom/tab-model-eval.git'
hint: Updates were rejected because the remote contains work that you do not
hint: have locally. This is usually caused by another repository pushing to
hint: the same ref. If you want to integrate the remote changes, use
hint: 'git pull' before pushing again.
hint: See the 'Note about fast-forwards' in 'git push --help' for details.
</stdout>

# Assistant
```bash
git pull
```

# User
<stdout>
remote: Enumerating objects: 13, done.
remote: Counting objects: 100% (13/13), done.
remote: Compressing objects: 100% (5/5), done.
remote: Total 5 (delta 2), reused 0 (delta 0), pack-reused 0 (from 0)
Unpacking objects: 100% (5/5), 1.18 KiB | 20.00 KiB/s, done.
From github.com:p-doom/tab-model-eval
   9cb411a..9d8e833  dev/ali    -> origin/dev/ali
Updating 9cb411a..9d8e833
Fast-forward
 src/input_pipeline/trace_to_testcases.py | 1 +
 1 file changed, 1 insertion(+)
</stdout>

# Assistant
```bash
git push
```

# User
<stdout>
Everything up-to-date
</stdout>
[tab] data/eval/handcrafted/git_push_complex.md (markdown)
# Assistant
```bash
ls -la
```

# User
<stdout>
total 712
drwxr-xr-x  10 mihir.mahajan mihir.mahajan    361 Oct 28 20:10 .
drwxr-xr-x  14 mihir.mahajan mihir.mahajan    441 Nov 28 12:49 ..
drwxr-xr-x   5 mihir.mahajan mihir.mahajan    166 Oct 20 09:39 data
drwxr-xr-x   4 mihir.mahajan mihir.mahajan     52 Oct 29 11:50 gifs
drwxr-xr-x   8 mihir.mahajan mihir.mahajan    354 Oct 29 11:45 .git
-rw-r--r--   1 mihir.mahajan mihir.mahajan     68 Oct  7 14:57 .gitignore
drwxr-xr-x   6 mihir.mahajan mihir.mahajan    364 Oct 28 20:10 jasmine
-rw-r--r--   1 mihir.mahajan mihir.mahajan  11357 Oct  7 14:57 LICENSE
-rw-r--r--   1 mihir.mahajan mihir.mahajan     90 Oct  7 14:57 .pre-commit-config.yaml
-rw-r--r--   1 mihir.mahajan mihir.mahajan    435 Oct  7 14:57 pyproject.toml
-rw-r--r--   1 mihir.mahajan mihir.mahajan  10143 Oct 28 20:10 README.md
drwxr-xr-x   8 mihir.mahajan mihir.mahajan    166 Oct  7 14:57 slurm
drwxr-xr-x   3 mihir.mahajan mihir.mahajan     58 Oct  7 14:57 tests
-rw-r--r--   1 mihir.mahajan mihir.mahajan 253777 Oct  7 15:05 uv.lock
drwxr-xr-x   5 mihir.mahajan mihir.mahajan    197 Oct  7 17:55 .venv
drwxr-xr-x 110 mihir.mahajan mihir.mahajan   5104 Oct 29 10:40 wandb
</stdout>

# Assistant
```bash
git status
```

# User
<stdout>
On branch chore/refactor-diffusion-branch
Your branch is up to date with 'origin/chore/refactor-diffusion-branch'.

Changes not staged for commit:
  (use "git add <file>..." to update what will be committed)
  (use "git restore <file>..." to discard changes in working directory)
        modified:   jasmine/baselines/diffusion/sample_diffusion.py
        modified:   jasmine/baselines/maskgit/sample_maskgit.py

Untracked files:
  (use "git add <file>..." to include in what will be committed)
        data/uv.lock
        slurm/
        uv.lock

no changes added to commit (use "git add" and/or "git commit -a")
</stdout>

# Assistant
```bash
git commit -am "Added sampling scripts for diffusion and maskgit baselines"
```

# User
<stdout>
black....................................................................Failed
- hook id: black
- files were modified by this hook

reformatted jasmine/baselines/diffusion/sample_diffusion.py
reformatted jasmine/baselines/maskgit/sample_maskgit.py

All done! ✨ 🍰 ✨
2 files reformatted.
</stdout>

# Assistant
```bash
git commit -am "Added sampling scripts for diffusion and maskgit baselines"
```

# User
<stdout>
black....................................................................Passed
[chore/refactor-diffusion-branch cf058bf] Added sampling scripts for diffusion and maskgit baselines
 2 files changed, 72 insertions(+)!
</stdout>

# Assistant
```bash
gi tpush
```

# User
<stdout>
bash: gi: command not found
</stdout>

# Assistant
```bash
git push
```
[tab] src/input_pipeline/validation_to_testcases copy.py (python)
import tyro
import os
from dataclasses import dataclass


@dataclass
class Args:
    input_file: str = "data/eval/val/validation.jsonl"
    output_file: str = "data/eval/val/validation_testcases.jsonl"
    task_name: str = "validation_set"


def convert_to_incremental_jsonl(line, output_file, task_name):
    data = json.loads(line)

    system_prompt = data.get("system_prompt", "")
    conversations = data.get("conversations", [])

    output_lines = []
    context = []

    for i, conv in enumerate(conversations):
        # Convert "Assistant"/"User" to "assistant"/"user"
        role = conv["from"].lower()
        content = conv["value"]

        # Add current message to context
        context.append({"role": role, "content": content})

        if i > 0 and role == "assistant":
            task_entry = {
                "system_prompt": system_prompt,
                "task_id": f"{task_name}/{i-1}",
                "context": context[:-1].copy(),
                "expected_final_response": content,
            }
            output_lines.append(task_entry)

    with open(output_file, "a") as f:
        for line in output_lines:
            f.write(json.dumps(line) + "
")

    print(f"Converted {len(output_lines)} entries to {output_file}")


if __name__ == "__main__":
    args = tyro.cli(Args)

    if os.path.exists(args.output_file):
        os.remove(args.output_file)

    # iterate over jsonl file and convert to incremental jsonl
    conversation_id = 0
    with open(args.input_file, "r") as f:
        for line in f:
            task_name = f"conversation_{conversation_id}/{args.task_name}"
            convert_to_incremental_jsonl(line, args.output_file, task_name)
            conversation_id += 1
[tab] src/input_pipeline/validation_to_testcases.py (python)
import json
import tyro
import os
from dataclasses import dataclass


@dataclass
class Args:
    input_file: str = "data/eval/val/validation.jsonl"
    output_file: str = "data/eval/val/validation_testcases.jsonl"
    task_name: str = "validation_set"


def convert_to_incremental_jsonl(line, output_file, task_name):
    data = json.loads(line)

    system_prompt = data.get("system_prompt", "")
    conversations = data.get("conversations", [])

    output_lines = []
    context = []

    for i, conv in enumerate(conversations):
        # Convert "Assistant"/"User" to "assistant"/"user"
        role = conv["from"].lower()
        content = conv["value"]

        # Add current message to context
        context.append({"role": role, "content": content})

        if i > 0 and role == "assistant":
            task_entry = {
                "system_prompt": system_prompt,
                "task_id": f"{task_name}/{i-1}",
                "context": context[:-1].copy(),
                "expected_final_response": content,
            }
            output_lines.append(task_entry)

    with open(output_file, "a") as f:
        for line in output_lines:
            f.write(json.dumps(line) + "
")

    print(f"Converted {len(output_lines)} entries to {output_file}")


if __name__ == "__main__":
    args = tyro.cli(Args)

    if os.path.exists(args.output_file):
        os.remove(args.output_file)

    # iterate over jsonl file and convert to incremental jsonl
    conversation_id = 0
    with open(args.input_file, "r") as f:
        for line in f:
            task_name = f"conversation_{conversation_id}/{args.task_name}"
            convert_to_incremental_jsonl(line, args.output_file, task_name)
            conversation_id += 1
[content] src/input_pipeline/trace_to_testcases.py (python)
            
            # Extract eval tag if present (will be in sections[index + 1] if it exists)
            eval_tag = None
            if index + 1 < len(sections) and sections[index + 1] in ("EVAL", "NO_EVAL"):
                eval_tag = sections[index + 1]
                content_index = index + 2
            else:
                content_index = index + 1
[content] src/input_pipeline/trace_to_testcases.py (python)
                test_case = {
                    "task_id": f"{task_name}/{task_number}",
                    "context": context,
                    "expected_final_response": expected_response,
                }
                test_cases.append(test_case)
                task_number += 1
[content] src/input_pipeline/trace_to_testcases.py (python)
        # Only create test cases for assistant turns marked with <EVAL>
        if i + 1 < len(parsed_data):
            assistant_turn = parsed_data[i + 1]
            if assistant_turn["role"] == "assistant":
                eval_tag = assistant_turn.get("eval_tag")
                # Skip if NO_EVAL or if eval_tag is None (old format without tags)
                if eval_tag == "NO_EVAL":
                    continue
                # Only create test case if EVAL tag is present
                if eval_tag != "EVAL":
                    continue
                
                context = parsed_data[: i + 1]
                expected_response = assistant_turn["content"]
[content] src/input_pipeline/trace_to_testcases.py (python)
        test_case = {
            "task_id": f"{task_name}/{task_number}",
            "context": context,
            "expected_final_response": expected_response,
        }
        test_cases.append(test_case)
        task_number += 1
[content] src/input_pipeline/trace_to_testcases.py (python)
    
    # Parse sections: pattern is [content, role, eval_tag, content, role, eval_tag, ...]
    # First section is usually empty or initial content, so we start from index 1
    i = 0
    while i < len(sections):
        # Check if current section is a role (User or Assistant)
        if sections[i] in ("User", "Assistant"):
            role = sections[i].lower()
            # Get eval_tag (next section if it exists and is EVAL/NO_EVAL, otherwise None)
            eval_tag = None
            if i + 1 < len(sections) and sections[i + 1] in ("EVAL", "NO_EVAL"):
                eval_tag = sections[i + 1]
                content_index = i + 2
            else:
                content_index = i + 1
            
            # Get content (next section after role/eval_tag)
            content = sections[content_index].strip() if content_index < len(sections) else ""
            
            parsed_data.append({
                "role": role,
                "content": content,
                "eval_tag": eval_tag,  # None for User, "EVAL" or "NO_EVAL" for Assistant
            })
            
            # Move to next role (skip content)
            i = content_index + 1
        else:
            # Skip non-role sections (shouldn't happen after filtering, but just in case)
            i += 1
[content] src/input_pipeline/trace_to_testcases.py (python)
    # Updated regex to capture eval tags: # Assistant <EVAL> or # Assistant <NO_EVAL>
    # re.split with capturing groups includes the groups in the result
    # Pattern: text, role, eval_tag, content, role, eval_tag, content, ...
    sections = re.split(r"# (User|Assistant)(?: <(EVAL|NO_EVAL)>)?", content)
    # Remove empty strings and strip
[content] src/input_pipeline/trace_to_testcases.py (python)
    current_dialogue = None

    for index, section in enumerate(sections):
        if section.startswith("User") or section.startswith("Assistant"):
            if current_dialogue is not None:
                parsed_data.append(current_dialogue)
            current_dialogue = {
                "role": section.lower(),
                "content": sections[index + 1].strip(),
            }

    if current_dialogue is not None:
        parsed_data.append(current_dialogue)
[content] src/input_pipeline/trace_to_testcases.py (python)
    last_pos = 0
    
    # Find all matches
    for match in re.finditer(pattern, content):
        # Get the role and eval_tag from the match
        role = match.group(1).lower()
        eval_tag = match.group(2)  # Will be None if not present
        
        # Find the content between this match and the next match (or end of file)
        start_pos = match.end()
        next_match = None
        for next_match_iter in re.finditer(pattern, content):
            if next_match_iter.start() > match.start():
                next_match = next_match_iter
                break
        
        if next_match:
            end_pos = next_match.start()
        else:
            end_pos = len(content)
        
        dialogue_content = content[start_pos:end_pos].strip()
        
        parsed_data.append({
            "role": role,
            "content": dialogue_content,
            "eval_tag": eval_tag,  # None for User, "EVAL" or "NO_EVAL" for Assistant
        })
        
        last_pos = end_pos
[content] src/input_pipeline/trace_to_testcases.py (python)
    # Use regex to find all headers with optional eval tags
    # Pattern matches: # User or # Assistant <EVAL> or # Assistant <NO_EVAL>
    pattern = r"# (User|Assistant)(?: <(EVAL|NO_EVAL)>)?
"
[content] src/input_pipeline/trace_to_testcases.py (python)
    
    # Find all matches
    for match in re.finditer(pattern, content):
        # Get the role and eval_tag from the match
        role = match.group(1).lower()
        eval_tag = match.group(2)  # Will be None if not present
        
        # Find the content between this match and the next match (or end of file)
        start_pos = match.end()
        next_match = None
        for next_match_iter in re.finditer(pattern, content):
            if next_match_iter.start() > match.start():
                next_match = next_match_iter
                break
        
        if next_match:
            end_pos = next_match.start()
        else:
            end_pos = len(content)
        
        dialogue_content = content[start_pos:end_pos].strip()
        
        parsed_data.append({
            "role": role,
            "content": dialogue_content,
            "eval_tag": eval_tag,  # None for User, "EVAL" or "NO_EVAL" for Assistant
        })
        
        last_pos = end_pos
[tab] slurm/dev/alfred/berlin/handcrafted/sglang_eval_server.sh (shellscript)
#! /bin/bash

module load CUDA/12.8
source .venv/bin/activate



model_path="Qwen/Qwen3-Coder-30B-A3B-Instruct"

accept_threshold=0.7
server_host="hai008"


python src/evaluation/sglang_eval.py \
  --judge_model_path $model_path \
  --accept_threshold $accept_threshold \
  --server_host $server_host \
  --judge_prompt_file_with_context data/prompts/command_evaluation_prompt_with_context.txt \
  --no_launch_server \
[tab] slurm/dev/alfred/berlin/handcrafted/sglang_generate_server.sh (shellscript)
#! /bin/bash

module load CUDA/12.8
source .venv/bin/activate


# model_path="Qwen/Qwen3-Coder-30B-A3B-Instruct"
# model_path="Qwen/Qwen3-0.6B"
# context_length=128000

server_host="hai005"
server_port=30000
model_path="/fast/project/HFMI_SynergyUnit/tab_model/data/outputs/hf/qwen3_600m_full_finetune_crowd_code_dataset_seqlen_16k_token_batch_size_128k/2025-11-24_18-17-40"
context_length=40960


python src/evaluation/sglang_generations.py \
  --model_path $model_path \
  --context_length $context_length \
  --server_host $server_host \
  --server_port $server_port \
  --concurrency 32 \
  --no_launch_server
[tab] pyproject.toml (plaintext)
[build-system]
requires = ["setuptools>=61.0", "wheel"]
build-backend = "setuptools.build_meta"

[project]
name = "tab-model-eval"
version = "0.1.0"
description = "Evaluation framework for tab models (and later, coding agents)"
readme = "README.md"
requires-python = ">=3.9"
license = {text = "MIT"}
authors = [
    {name = "Alfred Nguyen", email = "alfred@pdoom.org"}
]
keywords = ["llm", "evaluation", "coding-agent", "sglang"]
classifiers = [
    "Development Status :: 3 - Alpha",
    "Intended Audience :: Science/Research",
    "License :: OSI Approved :: MIT License",
    "Programming Language :: Python :: 3",
    "Programming Language :: Python :: 3.9",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
    "Programming Language :: Python :: 3.12",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
]

dependencies = [
    "sglang",
    "lm-eval",
    "tyro",
    "openai>=1.0.0",
    "httpx>=0.24.0",
    "wandb",
    "tqdm",
]

[project.optional-dependencies]
dev = [
    "black",
    "ruff",
    "mypy",
    "pytest",
]

[project.scripts]
# Add entry points here if you want to create CLI commands
# tab-model-eval = "src.evaluation.sglang_eval:main"

[tool.setuptools]
packages = ["src"]

[tool.setuptools.package-dir]
"" = "."

[tool.black]
line-length = 100
target-version = ["py39", "py310", "py311", "py312"]
include = '\.pyi?$'
extend-exclude = '''
/(
  # directories
  \.eggs
  | \.git
  | \.hg
  | \.mypy_cache
  | \.tox
  | \.venv
  | _build
  | buck-out
  | build
  | dist
)/
'''

[tool.ruff]
line-length = 100
target-version = "py39"
select = [
    "E",  # pycodestyle errors
    "W",  # pycodestyle warnings
    "F",  # pyflakes
    "I",  # isort
    "B",  # flake8-bugbear
    "C4", # flake8-comprehensions
    "UP", # pyupgrade
]
ignore = [
    "E501",  # line too long, handled by black
    "B008",  # do not perform function calls in argument defaults
    "C901",  # too complex
]

[tool.ruff.per-file-ignores]
"__init__.py" = ["F401"]

[tool.mypy]
python_version = "3.9"
warn_return_any = true
warn_unused_configs = true
disallow_untyped_defs = false
disallow_incomplete_defs = false
check_untyped_defs = true
no_implicit_optional = true
warn_redundant_casts = true
warn_unused_ignores = true
warn_no_return = true
follow_imports = "normal"
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = [
    "sglang.*",
    "lm_eval.*",
    "wandb.*",
]
ignore_missing_imports = true