p-doom/crowd-code-dataset-0.2 huggingface dataset:
[content] crowd-pilot/crowd-pilot/serialization_utils.py (python)
@dataclass
class ChunkState:
    """
    Mutable state used while constructing conversation chunks.
    """
    chunks: List[List[Dict[str, str]]]
    max_chars_per_conversation: Optional[int]
    current_chunk: List[Dict[str, str]] = field(default_factory=list)
    current_chars: int = 0
    files_opened_in_chunk: set[str] = field(default_factory=set)

    def start_new_chunk(self) -> None:
        if self.current_chunk:
            self.chunks.append(self.current_chunk)
        self.current_chunk = []
        self.current_chars = 0
        self.files_opened_in_chunk.clear()

    def append_turn(self, turn: Dict[str, str]) -> None:
        value = turn.get("value", "")
        turn_len = len(value)
        if (
            self.max_chars_per_conversation
            and self.max_chars_per_conversation > 0
            and self.current_chunk  # only split once we already have content
            and self.current_chars + turn_len > self.max_chars_per_conversation
        ):
            self.start_new_chunk()
        self.current_chunk.append(turn)
        self.current_chars += turn_len

    def ensure_file_open(
        self,
        file_path: str,
        file_states: Dict[str, str],
    ) -> None:
        """
        Ensure that the current chunk contains a full `cat -n` listing for
        the given file when operating on chunks after the first.
        """
        if not self.max_chars_per_conversation or self.max_chars_per_conversation <= 0:
            return
        # Only re-log files for chunks after the first; the first chunk
        # already sees initial file opens in-session.
        if not self.chunks:
            return
        if file_path in self.files_opened_in_chunk:
            return
        content = file_states.get(file_path, "")
        cmd = f"cat -n {file_path}"
        self.append_turn({
            "from": "Assistant",
            "value": _fenced_block("bash", _clean_text(cmd)),
        })
        output = _line_numbered_output(content)
        self.append_turn({
            "from": "User",
            "value": f"<stdout>
{output}
</stdout>",
        })
        self.files_opened_in_chunk.add(file_path)
[content] crowd-pilot/crowd-pilot/serialization_utils.py (python)
        nonlocal files_opened_in_chunk
        if not max_chars_per_conversation or max_chars_per_conversation <= 0:
            return
        # Only re-log files for chunks after the first; the first chunk
        # already sees initial file opens in-session.
        if not chunks:
            return
        if file_path in files_opened_in_chunk:
            return
        content = file_states.get(file_path, "")
        cmd = f"cat -n {file_path}"
        _append_turn({
            "from": "Assistant",
            "value": _fenced_block("bash", _clean_text(cmd)),
        })
        output = _line_numbered_output(content)
        _append_turn({
            "from": "User",
            "value": f"<stdout>
{output}
</stdout>",
        })
        files_opened_in_chunk.add(file_path)
[content] crowd-pilot/crowd-pilot/serialization_utils.py (python)
        nonlocal current_chunk, current_chars
        value = turn.get("value", "")
        turn_len = len(value)
        if (
            max_chars_per_conversation
            and max_chars_per_conversation > 0
            and current_chunk
            and current_chars + turn_len > max_chars_per_conversation
        ):
            _start_new_chunk()
        current_chunk.append(turn)
        current_chars += turn_len
[content] crowd-pilot/crowd-pilot/serialization_utils.py (python)
    def _start_new_chunk() -> None:
        nonlocal current_chunk, current_chars, files_opened_in_chunk
        if current_chunk:
            chunks.append(current_chunk)
        current_chunk = []
        current_chars = 0
        files_opened_in_chunk = set()
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/download_qwen_30b_3b_nemo.py (python)
from nemo.collections import llm

if __name__ == "__main__":
  llm.import_ckpt(model=llm.Qwen3Model(llm.Qwen3Config30B_A3B()), source='hf://Qwen/Qwen3-Coder-30B-A3B-Instruct', output_path="/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-coder-30b-a3b-instruct")
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/test_sft.py (python)
import nemo_run as run
from nemo.collections import llm

NAME = "qwen3_30b_a3b"

recipe = llm.qwen3_30b_a3b.finetune_recipe(
    name=NAME,
    dir=f"test_output",
    num_nodes=1,
    num_gpus_per_node=2,
    peft_scheme='lora',  # 'lora', 'none'
    packed_sequence=True,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_finetuning():
    recipe.resume.restore_config.path = "/fast/project/HFMI_SynergyUnit/tab_model/data/checkpoints/nemo_converted_weights_qwen3-coder-30b-a3b-instruct/"
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_finetuning()
[content] crowd-pilot/crowd-pilot/serialization_utils.py (python)
def _discover_local_sessions(root: Path) -> List[Path]:
    # Recursively find all CSV files
    paths: List[Path] = []
    for p in root.rglob("*.csv"):
        if p.is_file():
            paths.append(p)
    paths.sort()
    return paths


def _chunk_text(text: str, target_chars: int, overlap_chars: int) -> List[str]:
    """Split a long text into overlapping chunks near target length."""
    if target_chars <= 0:
        return [text]
    n = len(text)
    if n <= target_chars:
        return [text]

    chunks: List[str] = []
    start = 0
    # Ensure sane overlap
    overlap = max(0, min(overlap_chars, target_chars // 2))
    while start < n:
        end_target = min(start + target_chars, n)
        if end_target < n:
            end = end_target
        else:
            end = n
        chunk = text[start:end].strip()
        chunks.append(chunk)
        if end == n:
            break
        # advance with overlap
        start = max(0, end - overlap)
        if start >= n:
            break
    return chunks
[content] crowd-pilot/crowd-pilot/serialization_utils.py (python)
        # For a flushed edit burst, the sed command transforms `before_snapshot`
        # into `after_state`. Ensure the chunk contains a full-file view of the
        # *pre-edit* contents so the sed is semantically consistent.
        chunk_state.ensure_file_open(target_file, file_states, content_override=before_snapshot)
[content] crowd-pilot/crowd-pilot/serialization_utils.py (python)
        # For a flushed edit burst, the sed command transforms `before_snapshot`
        # into `after_state`. Ensure the chunk contains a full-file view of the
        # *pre-edit* contents so the sed is semantically consistent.
        chunk_state.ensure_file_open(target_file, before_snapshot)
[content] crowd-pilot/crowd-pilot/serialize_dataset_nemo_json.py (python)
    _print_turn_stats("all", chunk_turn_counts_all)
    _print_turn_stats("kept", chunk_turn_counts_kept)
    _print_char_stats("all", chunk_char_counts_all)
    _print_char_stats("kept", chunk_char_counts_kept)
[content] crowd-pilot/crowd-pilot/serialize_dataset_nemo_json.py (python)
        ]
        chunk_turn_counts_all.extend(per_chunk_turns)
        chunk_char_counts_all.extend(per_chunk_chars)

        # Aggregate per-session turns for filtering
        total_turns = sum(per_chunk_turns)
[content] crowd-pilot/crowd-pilot/serialize_dataset_nemo_json.py (python)
        # Per-chunk statistics (for reporting)
        per_chunk_turns = [len(chunk) for chunk in conversation_chunks]
        per_chunk_chars = [
            sum(len(turn.get("value", "")) for turn in chunk)
[content] crowd-pilot/crowd-pilot/serialize_dataset_nemo_json.py (python)
        sorted_vals = sorted(values)
        mid = count // 2
        if count % 2 == 1:
            median = float(sorted_vals[mid])
        else:
            median = 0.5 * (sorted_vals[mid - 1] + sorted_vals[mid])
        mode_val, mode_freq = Counter(values).most_common(1)[0]
[content] crowd-pilot/crowd-pilot/serialize_dataset_nemo_json.py (python)
        avg = total / count if count > 0 else 0.0
        sorted_vals = sorted(values)
        mid = count // 2
        if count % 2 == 1:
            median = float(sorted_vals[mid])
        else:
            median = 0.5 * (sorted_vals[mid - 1] + sorted_vals[mid])
[content] crowd-pilot/crowd-pilot/serialize_dataset_nemo_json.py (python)
p.add_argument("--shard_size", type=int, default=20000, help="(unused for JSONL)")
    p.add_argument("--overlap_chars", type=int, default=128, help="(unused for JSONL)")
    p.add_argument("--arrayrecord_group_size", type=int, default=1, help="(unused for JSONL)")
[terminal_command] TERMINAL ()
jq -r '  "mask: \(.mask)",  "system: \(.system)",  "conversations:",  (.conversations[] |    "  - from: \(.from)
    value:
\(.value)
---"  )' /fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/crowd-code-0.1/nemo_jsonl/val.jsonl | less
[content] crowd-pilot/crowd-pilot/serialize_dataset_nemo_json.py (python)
    # Persist metadata alongside the dataset files for downstream inspection.
    metadata = {
        "config": {
            "csv_root": cfg.csv_root,
            "output_dir": cfg.output_dir,
            "min_session_turns": cfg.min_session_turns,
            "max_docs": cfg.max_docs,
            "val_ratio": cfg.val_ratio,
            "target_chars_per_conversation": cfg.target_chars_per_conversation,
            "target_chars_per_turn": cfg.target_chars_per_turn,
        },
        "counts": {
            "total_sessions": total_sessions,
            "sessions_kept": sessions_kept,
            "skipped_short_sessions": skipped_short_sessions,
            "train_conversations": len(train_conversations),
            "val_conversations": len(val_conversations),
            "docs_written": docs_written,
        },
        "chunk_turn_stats": {
            "all": _compute_stats(chunk_turn_counts_all),
            "kept": _compute_stats(chunk_turn_counts_kept),
        },
        "chunk_char_stats": {
            "all": _compute_stats(chunk_char_counts_all),
            "kept": _compute_stats(chunk_char_counts_kept),
        },
        "files": {
            "train_path": str(train_path),
            "val_path": str(val_path),
        },
    }
    metadata_path = Path(cfg.output_dir) / "metadata.json"
    with open(metadata_path, "w", encoding="utf-8") as mf:
        json.dump(metadata, mf, ensure_ascii=False, indent=2)
    print(f"  Metadata: {metadata_path}")
[content] crowd-pilot/crowd-pilot/serialize_dataset_nemo_json.py (python)
        return {
            "count": count,
            "total": total,
            "median": median,
            "avg": avg,
            "min": min_len,
            "max": max_len,
        }

    def _print_turn_stats(label: str, values: List[int]) -> None:
        if not values:
            print(f"[debug] No {label} chunks for turn count stats.")
            return
        stats = _compute_stats(values)
        assert stats is not None
[tab] crowd-pilot/pyproject.toml (plaintext)
[project]
name = "crowd-pilot"
description = "Teaching language models to code like humans."
version = "0.1.0"
requires-python = ">=3.12"
dependencies = [
    "datasets>=2.19.0",
    "pandas>=2.2.2",
    "pyarrow>=15.0.2",
    "tqdm>=4.66.4",
    "regex>=2024.5.15",
    "array-record>=0.8.1",
    "tensorflow>=2.20.0",
    "hf-transfer>=0.1.9",
]

[project.urls]
Repository = "https://github.com/p-doom/crowd-pilot"