p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] crowd-pilot/crowd-pilot/test_nemo_format.py (python)
#!/usr/bin/env python3
"""
Quick test to verify NeMo JSON format generation
"""

import json
import pandas as pd
from serialization_utils import session_to_nemo_conversation

# Create a simple test session
test_data = {
    "Sequence": [1, 2, 3, 4],
    "Time": [0, 100, 200, 300],
    "File": ["test.py", "test.py", "test.py", "test.py"],
    "RangeOffset": [0, 0, 0, 0],
    "RangeLength": [0, 0, 0, 0],
    "Text": ["def hello():
    pass", None, "python test.py", "Hello World!"],
    "Language": ["python", "python", None, None],
    "Type": ["tab", "selection_command", "terminal_command", "terminal_output"]
}

df = pd.DataFrame(test_data)

# Convert to NeMo format
conversations = session_to_nemo_conversation(df)

# Create the full record
record = {
    "mask": "User",
    "system": "",
    "conversations": conversations
}

# Pretty print the result
print(json.dumps(record, indent=2, ensure_ascii=False))

print(f"
✓ Generated {len(conversations)} conversation turns")
print(f"✓ Format: NeMo 2.0 JSONL")
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/generate_jsonl_nemo_dataset.sh (shellscript)
#!/bin/bash

set -uex

OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/crowd-code-0.1/nemo_jsonl/"
CSV_ROOT="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/crowd-code-0.1/csv/"

uv run crowd-pilot/serialize_dataset_nemo_json.py --csv_root=$CSV_ROOT --output_dir=$OUTPUT_DIR
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/download_qwen_30b_3b_nemo.py (python)
from nemo.collections import llm

if __name__ == "__main__":
  llm.import_ckpt(model=llm.Qwen3Model(llm.Qwen3Config30B_A3B()), source='hf://Qwen/Qwen3-Coder-30B-A3B-Instruct', output_path="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/checkpoint_pretrained_nemo/Qwen/Qwen3-Coder-30B-A3B-Instruct")
[terminal_command] TERMINAL ()
mv /fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/checkpoint_pretrained_nemo/Qwen/Qwen3-Coder-30B-A3B-Instruct /fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights
[content] crowd-pilot/crowd-pilot/serialize_dataset_nemo_json.py (python)
            f"count={count}, avg_turns={avg:.1f}, min_turns={min_len}, max_turns={max_len}"
        )

    def _print_char_stats(label: str, values: List[int]) -> None:
        if not values:
            print(f"[debug] No {label} sessions for character stats.")
            return
        count = len(values)
        total = sum(values)
        avg = total / count
        min_len = min(values)
        max_len = max(values)
        print(
            f"[debug] {label.capitalize()} sessions char stats: "
            f"count={count}, avg_chars={avg:.1f}, min_chars={min_len}, max_chars={max_len}"
[content] crowd-pilot/crowd-pilot/serialize_dataset_nemo_json.py (python)
        # Character count for this session (sum over all turn values)
        session_chars = sum(len(turn.get("value", "")) for turn in conversations)
        session_char_counts_all.append(session_chars)