p-doom/crowd-code-dataset-0.2 huggingface dataset:
[content] miles/utils/mask_utils.py (python)


    def get_text_from_loss_mask(self, token_ids: list[int], loss_masks: list[int]) -> list[str]:
        selected_texts = []
        current_tokens = []

        for idx, mask in enumerate(loss_masks):
            if mask == 1:
                current_tokens.append(token_ids[idx])
            elif current_tokens:
                selected_texts.append(self.tokenizer.decode(current_tokens))
                current_tokens = []

        if current_tokens:
            selected_texts.append(self.tokenizer.decode(current_tokens))

        return selected_texts
[tab] slurm/jobs/mihir/berlin/crowd-pilot/run_sqwen3-0.6b-sft-torch-pythonic.sh (shellscript)
#!/usr/bin/env bash

#SBATCH --nodes=1
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:2
#SBATCH --mem=600GB
#SBATCH --signal=USR1@60
#SBATCH --output=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-sft-torch/%x_%j.log
#SBATCH --error=/fast/project/HFMI_SynergyUnit/mihir/logs/train/qwen3-0.6b-sft-torch/%x_%j.log
#SBATCH --job-name=run_qwen3-0.6b-sft-torch
#SBATCH --qos=low

trap 'echo "Job killed by signal"' USR1

cat $0
module load CUDA/12.8

source .venv/bin/activate

srun sh scripts/run-sft-torchrun.sh
[tab] miles/utils/processing_utils.py (python)
import base64
import io
import logging

from transformers import AutoProcessor, AutoTokenizer, PreTrainedTokenizerBase, ProcessorMixin

logger = logging.getLogger(__name__)


def load_tokenizer(name_or_path: str, **kwargs):
    return AutoTokenizer.from_pretrained(name_or_path, **kwargs)


def load_processor(name_or_path: str, **kwargs):
    try:
        proc = AutoProcessor.from_pretrained(name_or_path, **kwargs)
    except (OSError, ValueError) as e:
        logger.warning(f"Failed to load processor from {name_or_path}: {e}")
        proc = None

    # If HF returned a tokenizer, discard it.
    if isinstance(proc, PreTrainedTokenizerBase) or not isinstance(proc, ProcessorMixin):
        proc = None

    return proc


def prepare_model_inputs(prompt, tokenizer, processor=None, metadata=None, apply_chat_template_kwargs=None):
    """Prepare all inputs for model inference.

    Returns:
        tuple: (input_ids, extra_info)
            - input_ids: Token IDs for the prompt
            - extra_info: Dict with 'images', 'videos', 'multimodal_inputs' (or empty dict)
    """
    tools = metadata.get("tools") if metadata else None
    text_prompt = tokenizer.apply_chat_template(
        prompt,
        tools=tools,
        tokenize=False,
        add_generation_prompt=True,
        **(apply_chat_template_kwargs or {}),
    )

    if not processor:
        input_ids = tokenizer.encode(text_prompt, add_special_tokens=False)
        return input_ids, {}
    else:
        # temporary solution, will write image utils for miles later
        from qwen_vl_utils import process_vision_info

        images, videos = process_vision_info(prompt)

        # Get input IDs with full prompt (text + multimodal)
        processor_output = processor(text=text_prompt, images=images, videos=videos)
        input_ids = processor_output["input_ids"][0]

        # Extract multimodal tokens (exclude text-related tokens)
        multimodal_inputs = {k: v for k, v in processor_output.items() if k not in ["input_ids", "attention_mask"]}

        extra_info = {
            "images": images,
            "videos": videos,
            "multimodal_inputs": multimodal_inputs,
        }

        return input_ids, extra_info


def encode_image_for_rollout_engine(image) -> str:
    """Load an image from path, ensure RGB, encode as JPEG base64 string."""
    buffer = io.BytesIO()
    if image.mode != "RGB":
        image = image.convert("RGB")
    image.save(buffer, format="PNG")
    return base64.b64encode(buffer.getvalue()).decode("utf-8")