Delete train_resonate.py with huggingface_hub

train_resonate.py

@@ -1,215 +0,0 @@
-"""
-Train Gemma-3 270M-IT with LoRA for /resonate/ format.
-Freeze embed_tokens (63% of model = all 140 languages preserved).
-LoRA rank 16 on Q+V projections only — minimal intervention.
-"""
-
-import json, os, sys, time, random, math
-import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM
-from peft import LoraConfig, get_peft_model, TaskType
-
-# --- config ---
-MODEL = 'unsloth/gemma-3-270m-it'
-RANK = 16
-ALPHA = 32
-LR = 2e-4
-EPOCHS = 3
-BATCH = 4
-GRAD_ACCUM = 4  # effective batch 16
-MAX_LEN = 1024
-EVAL_EVERY = 100
-SAVE_DIR = 'gemma3-resonate'
-
-# --- load data ---
-print('[data] Loading...')
-data = []
-for path in ['resonance_yent_full.jsonl', 'resonance_gold_10.jsonl']:
-    if os.path.exists(path):
-        with open(path) as f:
-            for line in f:
-                d = json.loads(line)
-                data.append(d)
-print(f'[data] {len(data)} examples')
-random.seed(42)
-random.shuffle(data)
-
-split = int(len(data) * 0.95)
-train_data = data[:split]
-val_data = data[split:]
-print(f'[data] train={len(train_data)}, val={len(val_data)}')
-
-# --- load model ---
-print('[model] Loading Gemma-3 270M-IT...')
-tokenizer = AutoTokenizer.from_pretrained(MODEL)
-model = AutoModelForCausalLM.from_pretrained(MODEL, dtype=torch.bfloat16).cuda()
-
-n_total = sum(p.numel() for p in model.parameters())
-n_embed = sum(p.numel() for n, p in model.named_parameters() if 'embed_tokens' in n)
-print(f'[model] {n_total/1e6:.1f}M total, {n_embed/1e6:.1f}M in embed_tokens ({n_embed*100/n_total:.0f}%)')
-
-# --- LoRA config ---
-lora_config = LoraConfig(
-    task_type=TaskType.CAUSAL_LM,
-    r=RANK,
-    lora_alpha=ALPHA,
-    lora_dropout=0.05,
-    target_modules=['q_proj', 'v_proj'],
-    bias='none',
-)
-
-model = get_peft_model(model, lora_config)
-trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
-frozen = sum(p.numel() for p in model.parameters() if not p.requires_grad)
-print(f'[lora] trainable={trainable/1e6:.2f}M ({trainable*100/n_total:.1f}%), frozen={frozen/1e6:.1f}M')
-
-# --- prepare data ---
-def format_example(d):
-    msgs = d['messages']
-    text = ''
-    for m in msgs:
-        if m['role'] == 'user':
-            text += f"<start_of_turn>user\n{m['content']}<end_of_turn>\n"
-        elif m['role'] == 'assistant':
-            text += f"<start_of_turn>model\n{m['content']}<end_of_turn>\n"
-    return text
-
-def tokenize_with_labels(text):
-    toks = tokenizer(text, truncation=True, max_length=MAX_LEN, return_tensors='pt')
-    input_ids = toks['input_ids'][0]
-    labels = input_ids.clone()
-    # mask user turn — only train on model output
-    model_marker = '<start_of_turn>model\n'
-    idx = text.find(model_marker)
-    if idx > 0:
-        prefix = text[:idx + len(model_marker)]
-        prefix_toks = tokenizer(prefix, add_special_tokens=False)['input_ids']
-        mask_len = min(len(prefix_toks), len(labels))
-        labels[:mask_len] = -100
-    return input_ids, labels
-
-print('[data] Tokenizing...')
-train_tokens = []
-for d in train_data:
-    text = format_example(d)
-    ids, labels = tokenize_with_labels(text)
-    if len(ids) > 10:
-        train_tokens.append((ids, labels))
-
-val_tokens = []
-for d in val_data:
-    text = format_example(d)
-    ids, labels = tokenize_with_labels(text)
-    if len(ids) > 10:
-        val_tokens.append((ids, labels))
-
-print(f'[data] {len(train_tokens)} train, {len(val_tokens)} val tokenized')
-if train_tokens:
-    avg_len = sum(len(t[0]) for t in train_tokens) / len(train_tokens)
-    print(f'[data] avg length: {avg_len:.0f} tokens')
-
-# --- training ---
-optimizer = torch.optim.AdamW(
-    [p for p in model.parameters() if p.requires_grad],
-    lr=LR, weight_decay=0.01
-)
-
-total_steps = len(train_tokens) * EPOCHS // (BATCH * GRAD_ACCUM)
-warmup_steps = int(total_steps * 0.1)
-print(f'[train] {total_steps} steps, {warmup_steps} warmup, {EPOCHS} epochs')
-
-def get_lr(step):
-    if step < warmup_steps:
-        return LR * step / max(warmup_steps, 1)
-    progress = (step - warmup_steps) / max(total_steps - warmup_steps, 1)
-    return LR * 0.5 * (1 + math.cos(math.pi * progress))
-
-model.train()
-step = 0
-best_val_loss = float('inf')
-os.makedirs(SAVE_DIR, exist_ok=True)
-t0 = time.time()
-
-for epoch in range(EPOCHS):
-    random.shuffle(train_tokens)
-    epoch_loss = 0
-    epoch_count = 0
-    optimizer.zero_grad()
-
-    for i, (ids, labels) in enumerate(train_tokens):
-        ids = ids.unsqueeze(0).cuda()
-        labels = labels.unsqueeze(0).cuda()
-
-        outputs = model(input_ids=ids, labels=labels)
-        loss = outputs.loss / GRAD_ACCUM
-        loss.backward()
-
-        epoch_loss += outputs.loss.item()
-        epoch_count += 1
-
-        if (i + 1) % GRAD_ACCUM == 0:
-            step += 1
-            lr = get_lr(step)
-            for g in optimizer.param_groups:
-                g['lr'] = lr
-
-            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
-            optimizer.step()
-            optimizer.zero_grad()
-
-            if step % 50 == 0:
-                avg = epoch_loss / epoch_count
-                elapsed = time.time() - t0
-                print(f'  ep{epoch+1} step {step}/{total_steps} | train loss {avg:.4f} | lr {lr:.6f} | {elapsed:.0f}s', flush=True)
-
-            if step % EVAL_EVERY == 0 and val_tokens:
-                model.eval()
-                val_loss = 0
-                with torch.no_grad():
-                    for vid, vlbl in val_tokens[:50]:
-                        vid = vid.unsqueeze(0).cuda()
-                        vlbl = vlbl.unsqueeze(0).cuda()
-                        out = model(input_ids=vid, labels=vlbl)
-                        val_loss += out.loss.item()
-                val_loss /= min(50, len(val_tokens))
-                print(f'  >>> VAL loss {val_loss:.4f} (best {best_val_loss:.4f})', flush=True)
-
-                if val_loss < best_val_loss:
-                    best_val_loss = val_loss
-                    model.save_pretrained(f'{SAVE_DIR}/best')
-                    tokenizer.save_pretrained(f'{SAVE_DIR}/best')
-                    print(f'  >>> SAVED best', flush=True)
-
-                model.train()
-
-    avg = epoch_loss / max(epoch_count, 1)
-    print(f'[epoch {epoch+1}] avg loss {avg:.4f}', flush=True)
-
-model.save_pretrained(f'{SAVE_DIR}/final')
-tokenizer.save_pretrained(f'{SAVE_DIR}/final')
-print(f'[done] best val loss: {best_val_loss:.4f}')
-
-# --- test generation ---
-print('\n[gen] Testing on 5 languages...')
-model.eval()
-
-prompts = [
-    'What is the meaning of life?',
-    'Explain recursion simply.',
-    'Dis-moi quelque chose en francais',
-    'Was denkst du ueber die Zukunft?',
-    'Why do programmers mass delete repos at 3am?',
-]
-
-for p in prompts:
-    text = f'<start_of_turn>user\n{p}<end_of_turn>\n<start_of_turn>model\n'
-    ids = tokenizer(text, return_tensors='pt').input_ids.cuda()
-    with torch.no_grad():
-        out = model.generate(ids, max_new_tokens=200, do_sample=True, temperature=0.7, top_k=40)
-    gen = tokenizer.decode(out[0], skip_special_tokens=True)
-    answer = gen.split('model\n')[-1] if 'model\n' in gen else gen[-300:]
-    print(f'\n>>> {p}')
-    print(answer[:300])
-    print('---')
-
-print(f'\n[done] Total time: {time.time()-t0:.0f}s')