p-doom/crowd-code-dataset-0.2 huggingface dataset:
[tab] slurm/dev/franz/berlin/crowd-pilot/nemo/test_sft.py (python)
import nemo_run as run
from nemo.collections import llm

NAME = "qwen3_30b_a3b"

recipe = llm.qwen3_30b_a3b.finetune_recipe(
    name=NAME,
    dir=f"/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/checkpoint_pretrained_nemo/Qwen/Qwen3-Coder-30B-A3B-Instruct",
    num_nodes=1,
    num_gpus_per_node=8,
    peft_scheme='lora',  # 'lora', 'none'
    packed_sequence=True,
)

def local_executor_torchrun(nodes: int, devices: int) -> run.LocalExecutor:
    # Env vars for jobs are configured here
    env_vars = {
        "TORCH_NCCL_AVOID_RECORD_STREAMS": "1",
        "NCCL_NVLS_ENABLE": "0",
        "NVTE_DP_AMAX_REDUCE_INTERVAL": "0",
        "NVTE_ASYNC_AMAX_REDUCTION": "1",
    }

    executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars)

    return executor

def run_pretraining():
    executor = local_executor_torchrun(nodes=recipe.trainer.num_nodes, devices=recipe.trainer.devices)

    run.run(recipe, executor=executor, name=NAME)

# This condition is necessary for the script to be compatible with Python's multiprocessing module.
if __name__ == "__main__":
    run_pretraining()
[tab] crowd-pilot/maxtext/docs/reference/terminology.md (markdown)
<!--
 Copyright 2024 Google LLC

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      https://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 -->

# Terminology

- **FLOP**: Floating Point Operation
- **FLOPS**: Plural form of FLOP
- **FLOP/s** or **FLOPs**: stands for Floating Point Operations Per Second.
- **MFU**: Model FLOP/s Utilization
- **ICI**: Interchip-interconnect.
- **HBM**: High Bandwidth Memory. Built with DRAM technology. Each chip usually has XX GiBs of HBM.
- **VMEM**: Vector Memory. Built with SRAM technology. Each chip usually has XX MiBs of VMEM.
- **DCN**: Data Center Network 
- **PCIe**: Peripheral Component Interconnect Express. How the TPUs communicate with the CPU.
- **AI**: Arithmetic Intensity
- **Rank**: Position or ID of a worker within a group of workers. This is not the same as Rank in linear algebra.
[content] slurm/dev/franz/berlin/crowd-pilot/nemo/generate_jsonl_nemo_dataset.sh (shellscript)
#!/bin/bash

set -uex

OUTPUT_DIR="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/crowd-code-0.1/bash_format_array_record/"
CSV_ROOT="/fast/project/HFMI_SynergyUnit/jafar_ws/data/crowd-pilot/crowd-code-0.1/csv/"

uv run crowd-pilot/serialize_dataset_array_record.py --csv_root=$CSV_ROOT --output_dir=$OUTPUT_DIR
[content] crowd-pilot/crowd-pilot/serialization_utils.py (python)

                region_end = start_line_current + max(lines_added, lines_deleted, 0)
                # Flush pending edits if this edit is far from the pending region
                current_region = pending_edit_regions.get(file_path)
                if current_region is not None:
                    rstart, rend = current_region
                    if region_start < (rstart - coalesce_radius) or region_start > (rend + coalesce_radius):
                        _flush_pending_edit_for_file(file_path)
                        current_region = None
                after = _app
[content] crowd-pilot/crowd-pilot/serialization_utils.py (python)
 _line_numbered_output(content, vstart, vend)
                        parts.append(f"<stdout>
{viewport_output}
</stdout>")

            case "content":
                _flush_terminal_output_buffer()
                offset = int(row[
[content] crowd-pilot/crowd-pilot/serialization_utils.py (python)
ius: int = 10,
    normalize_terminal_output: bool = True,
    coalesce_radius: int = 5,
) -> str:
    r"""
    Serialize a session to a bash-like transcript comprised of:
      - Commands (bash fenced blocks): cat -n, sed -i 'S,Ec\...' && cat -n | sed -n 'VSTART,VENDp'
      - Outputs (<stdout>...</stdout>) that reflect the file state after each action
    Tracks per-file state and a per-file viewport. Viewport only shifts when selection moves out of bounds
    or when first initialized.
    """
    file_states: Dict[str, str] = {}
    per_file_viewport: Dict[str, Optional[Tuple[int, int]]] = {}

    parts: List[str] = []
    terminal_output_buffer: List[str] = []
    pending_edits_before: Dict[str, Optional[str]] = {}
    pending_edit_regions: Dict[str, Optional[Tuple[int, int]]] = {}

    def _flush_terminal_output_buffer() -> None:
        if not terminal_output_buffer:
            return
        aggregated = "".join(terminal_output_buffer)
        out = aggregated
        if normalize_terminal_output:
            out = _normalize_terminal_output(out)
        cleaned = _clean_text(out)
        if cleaned.strip():
            parts.append(f"<stdout>
{cleaned}
</stdout>")
        terminal_output_buffer.clear()

    def _flush_pending_edit_for_file(target_file: str) -> None:
        before_snap
[content] crowd-pilot/crowd-pilot/serialization_utils.py (python)


        # Long pause detection
        if last_time_ms is not None:
            delta = event_time - last_time_ms
            if delta > long_pause_threshold_ms:
                # TODO (f.srambical): think about whether we want to emit this as an observation or not
                parts.append(f"<obs long_pause ms=\"{delta}\" />")
        last_time_ms = event_time
[content] crowd-pilot/crowd-pilot/serialization_utils.py (python)

    p.add_argument("--shard_size", type=int, default=20000, help="Rows per shard (currently one session per shard)")


def session_to_nemo_conversations(



def load_hf_csv(hf_path: str, split: str) -> Dataset:
    loaded = load_dataset(hf_path, split=split)

    assert isinstance(loaded, Dataset), "Expected a Dataset from load_dataset"
    return loaded


def _discover_local_sessions(root: Path) -> List[Path]:
    # Recursively find all CSV files
    paths: List[Path] = []
    for p in root.rglob("*.csv"):
        if p.is_file():
            paths.append(p)
    paths.sort()
    return paths


def _chunk_text(text: str, target_chars: int, overlap_chars: int) -> List[str]:
    """Split a long text into overlapping chunks near target length."""
    if target_chars <= 0:
        return [text]
    n = len(text)
    if n <= target_chars:
        return [text]

    chunks: List[str] = []
    start = 0
    # Ensure sane overlap
    overlap = max(0, min(overlap_chars, target_chars // 2))
    while start < n:
        end_target = min(start + target_chars, n)
        if end_target < n:
            end = end_target
        else:
            end = n
        chunk = text[start:end].strip()
        chunks.append(chunk)
        if end == n:
            break
        # advance with overlap
        start = max(0, end - overlap)
        if start >= n:
            break
    return chunks
[content] crowd-pilot/crowd-pilot/serialization_utils.py (python)
) -> str:
    r"""
    Serialize a session to a bash-like transcript comprised of:
      - Commands (bash fenced blocks): cat -n, sed -i 'S,Ec\...' && cat -n | sed -n 'VSTART,VENDp'
      - Outputs (<stdout>...</stdout>) that reflect the file state after each action
    Tracks per-file state and a per-file viewport. Viewport only shifts when selection moves out of bounds
    or when first initialized.
[content] crowd-pilot/crowd-pilot/serialization_utils.py (python)
    def _emit_bash_command(command: str) -> None:
        """Emit a bash command as an Assistant turn."""
        cleaned = _clean_text(command)
        if cleaned.strip():
            conversations.append({
                "from": "Assistant",
                "value": f"```bash
{cleaned}
```"
            })
[content] crowd-pilot/crowd-pilot/serialization_utils.py (python)
) -> List[Dict[str, str]]:
    """
    Convert a session DataFrame directly to NeMo conversation format.
    
    Returns a list of conversation turns:
    [
        {"from": "Assistant", "value": "```bash\
command\
```"},
        {"from": "User", "value": "<stdout>\
output\
</stdout>"},
        ...
    ]