train(grpo): unified hint prompt, no-history chat, positive-advantage filter

- env: surface audience_active_hours + competitor_recent_post_hours in obs metadata
- prompt: single audience-hours hint, same for train + eval (clean delta = LoRA only)
- runner: drop assistant history (kills 4712-tok bloat); never append synthetic rest into training pairs; carry step idx for return back-up
- decode: greedy at eval, sampled (T=1.0, top_p=0.95) at rollout
- filter: positive group-relative advantage only; QUALITY_FLOOR=0.40 skips bad rounds
- LoRA: r=8 attn-only; lr 5e-6, 1 epoch, max_len 2048 (less drift)

Made-with: Cursor

server/viraltest_environment.py

@@ -1097,6 +1097,19 @@ class ViraltestEnvironment(Environment):
         if grader_score is not None:
             meta["grader_score"] = round(grader_score, 4)
 
+        audience_hours: set = set()
+        for seg in _AUDIENCE_DATA.get("segments", []):
+            audience_hours.update(seg.get("active_hours", []))
+        meta["audience_active_hours"] = sorted(audience_hours)
+
+        comp_hours = [
+            (self._hour - p["hours_ago"]) % 24
+            for comp in self._competitors
+            for p in comp.recent_posts
+            if p["hours_ago"] < 48
+        ]
+        meta["competitor_recent_post_hours"] = sorted(comp_hours)
+
         burnout_risk = min(1.0, self._low_energy_days / 5.0)
 
         return ViraltestObservation(

training/train_grpo.ipynb

@@ -400,7 +400,7 @@
       "metadata": {},
       "source": [
         "# Cell 8: LLM agent functions\n",
-        "SYSTEM_PROMPT = textwrap.dedent(\"\"\"\\\n",
+        "_SYSTEM_BASE = textwrap.dedent(\"\"\"\\\n",
         "You are an Instagram content strategy agent. Each step is one day.\n",
         "You manage a creator account over a 15-day cycle.\n",
         "\n",
@@ -439,6 +439,12 @@
         "- topic:        free-form string\n",
         "- empty scheduled_actions = full day rest\"\"\")\n",
         "\n",
+        "SYSTEM_PROMPT = _SYSTEM_BASE + textwrap.dedent(\"\"\"\n",
+        "\n",
+        "HINT: schedule posts during/just before the audience_active_hours window — that is when your target users are online.\"\"\")\n",
+        "SYSTEM_PROMPT_EVAL = SYSTEM_PROMPT\n",
+        "SYSTEM_PROMPT_TRAIN = SYSTEM_PROMPT\n",
+        "\n",
         "\n",
         "def format_obs(obs):\n",
         "    days = [\"Mon\", \"Tue\", \"Wed\", \"Thu\", \"Fri\", \"Sat\", \"Sun\"]\n",
@@ -449,6 +455,9 @@
         "        signals_str = (f\"Signals: watch={signals.watch_time:.3f} \"\n",
         "                       f\"sends={signals.sends_per_reach:.3f} \"\n",
         "                       f\"saves={signals.saves:.3f}\\n\")\n",
+        "    meta = getattr(obs, \"metadata\", None) or {}\n",
+        "    aud = meta.get(\"audience_active_hours\") or []\n",
+        "    comp = meta.get(\"competitor_recent_post_hours\") or []\n",
         "    tool_str = \"\"\n",
         "    for tr in getattr(obs, \"tool_results\", []):\n",
         "        if tr.success:\n",
@@ -459,8 +468,10 @@
         "            f\"Energy: {obs.creator_energy:.2f} | Followers: {obs.follower_count}\\n\"\n",
         "            f\"Engagement: {obs.engagement_rate:.3f} | Queue: {obs.content_queue_size}\\n\"\n",
         "            f\"{signals_str}\"\n",
+        "            f\"audience_active_hours: {aud}\\n\"\n",
+        "            f\"competitor_recent_post_hours: {comp}\\n\"\n",
         "            f\"Tool results:\\n{tool_str}\"\n",
-        "            f\"Plan your actions (JSON only):\")\n",
+        "            f\"Plan today's actions (JSON only):\")\n",
         "\n",
         "\n",
         "def is_well_formed_response(text):\n",
@@ -527,35 +538,37 @@
         "    return torch.device(\"cpu\")\n",
         "\n",
         "\n",
-        "def _build_chat(history, prompt):\n",
-        "    msgs = [{\"role\": \"system\", \"content\": SYSTEM_PROMPT}]\n",
-        "    msgs.extend(history[-14:])\n",
-        "    msgs.append({\"role\": \"user\", \"content\": prompt})\n",
-        "    return msgs\n",
+        "def _build_chat(system, prompt):\n",
+        "    return [\n",
+        "        {\"role\": \"system\", \"content\": system},\n",
+        "        {\"role\": \"user\", \"content\": prompt},\n",
+        "    ]\n",
         "\n",
         "\n",
-        "def _batched_generate(mdl, tok, prompts, temperature=0.7, max_new_tokens=512):\n",
+        "def _batched_generate(mdl, tok, prompts, eval=False, max_new_tokens=512):\n",
         "    enc = tok(prompts, return_tensors=\"pt\", padding=True, truncation=False).to(_infer_model_device(mdl))\n",
+        "    gen_kwargs = dict(max_new_tokens=max_new_tokens, pad_token_id=tok.pad_token_id)\n",
+        "    if eval:\n",
+        "        gen_kwargs.update(do_sample=False)\n",
+        "    else:\n",
+        "        gen_kwargs.update(do_sample=True, temperature=1.0, top_p=0.95)\n",
         "    with torch.no_grad():\n",
-        "        out = mdl.generate(\n",
-        "            **enc, max_new_tokens=max_new_tokens, temperature=temperature,\n",
-        "            do_sample=True, top_p=0.9, pad_token_id=tok.pad_token_id,\n",
-        "        )\n",
+        "        out = mdl.generate(**enc, **gen_kwargs)\n",
         "    resps = tok.batch_decode(out[:, enc[\"input_ids\"].shape[1]:], skip_special_tokens=True)\n",
         "    return resps, enc[\"input_ids\"].shape[1]\n",
         "\n",
         "\n",
-        "def run_llm_episodes_batched(mdl, tok, tasks_seeds, verbose=True):\n",
+        "def run_llm_episodes_batched(mdl, tok, tasks_seeds, verbose=True, eval=False, system=None):\n",
         "    \"\"\"Run N episodes in parallel. tasks_seeds: list of (task, seed). One batched generate per day.\"\"\"\n",
+        "    sys_prompt = system or (SYSTEM_PROMPT_EVAL if eval else SYSTEM_PROMPT_TRAIN)\n",
         "    n = len(tasks_seeds)\n",
         "    envs = [ViraltestEnvironment() for _ in range(n)]\n",
         "    obss = [envs[i].reset(task=t, seed=s) for i, (t, s) in enumerate(tasks_seeds)]\n",
-        "    histories = [[] for _ in range(n)]\n",
         "    rewards = [[] for _ in range(n)]\n",
         "    energies = [[obs.creator_energy] for obs in obss]\n",
         "    pairs = [[] for _ in range(n)]\n",
         "    done_mask = [obs.done for obs in obss]\n",
-        "    rest_resp = '{\"scheduled_actions\": []}'\n",
+        "    rest_action = ViraltestAction(scheduled_actions=[])\n",
         "\n",
         "    for day in range(1, TASK_HORIZON + 1):\n",
         "        active = [i for i in range(n) if not done_mask[i] and obss[i].creator_energy > 0.25]\n",
@@ -563,33 +576,26 @@
         "        if not active and not rest:\n",
         "            break\n",
         "\n",
-        "        resps_by_idx = {}\n",
+        "        actions_by_idx = {i: rest_action for i in rest}\n",
         "        if active:\n",
         "            prompts = [format_obs(obss[i]) for i in active]\n",
-        "            chats = [_build_chat(histories[i], p) for i, p in zip(active, prompts)]\n",
+        "            chats = [_build_chat(sys_prompt, p) for p in prompts]\n",
         "            texts = [tok.apply_chat_template(c, tokenize=False, add_generation_prompt=True) for c in chats]\n",
-        "            resps, ptok = _batched_generate(mdl, tok, texts)\n",
+        "            resps, ptok = _batched_generate(mdl, tok, texts, eval=eval)\n",
         "            if verbose:\n",
         "                print(f\"  D{day:2d}: batch={len(active)} rest={len(rest)} prompt_tok={ptok}\")\n",
         "            for j, i in enumerate(active):\n",
-        "                resps_by_idx[i] = (resps[j], prompts[j])\n",
-        "        for i in rest:\n",
-        "            resps_by_idx[i] = (rest_resp, format_obs(obss[i]))\n",
+        "                actions_by_idx[i] = parse_model_output(resps[j])\n",
+        "                pairs[i].append({\"prompt\": prompts[j], \"response\": resps[j],\n",
+        "                                 \"step\": len(rewards[i])})\n",
         "\n",
         "        for i in range(n):\n",
-        "            if done_mask[i] or i not in resps_by_idx:\n",
+        "            if done_mask[i] or i not in actions_by_idx:\n",
         "                continue\n",
-        "            resp, prompt = resps_by_idx[i]\n",
-        "            action = parse_model_output(resp)\n",
-        "            pairs[i].append({\"prompt\": prompt, \"response\": resp})\n",
-        "            obss[i] = envs[i].step(action)\n",
+        "            obss[i] = envs[i].step(actions_by_idx[i])\n",
         "            r = obss[i].reward or 0.0\n",
         "            rewards[i].append(r)\n",
         "            energies[i].append(obss[i].creator_energy)\n",
-        "            histories[i].extend([\n",
-        "                {\"role\": \"user\", \"content\": prompt},\n",
-        "                {\"role\": \"assistant\", \"content\": resp},\n",
-        "            ])\n",
         "            if obss[i].done:\n",
         "                done_mask[i] = True\n",
         "\n",
@@ -602,8 +608,9 @@
         "        for t in reversed(range(len(rewards[i]))):\n",
         "            G = rewards[i][t] + GAMMA * G\n",
         "            rets[t] = G\n",
-        "        for k, pr in enumerate(pairs[i]):\n",
-        "            pr[\"return\"] = rets[k] if k < len(rets) else 0.0\n",
+        "        for pr in pairs[i]:\n",
+        "            k = pr.get(\"step\", 0)\n",
+        "            pr[\"return\"] = rets[k] if 0 <= k < len(rets) else 0.0\n",
         "        results.append({\n",
         "            \"task\": task, \"seed\": seed, \"grader_score\": gs,\n",
         "            \"total_reward\": sum(rewards[i]), \"final_energy\": obss[i].creator_energy,\n",
@@ -641,7 +648,7 @@
         "print(\"=\" * 60)\n",
         "\n",
         "t0 = time.time()\n",
-        "results = run_llm_episodes_batched(model, tokenizer, [(t, 42) for t in TASKS], verbose=True)\n",
+        "results = run_llm_episodes_batched(model, tokenizer, [(t, 42) for t in TASKS], verbose=True, eval=True)\n",
         "before_results = {r[\"task\"]: r for r in results}\n",
         "\n",
         "print(\"\\n\" + \"=\" * 60)\n",
@@ -675,9 +682,8 @@
         "from peft import LoraConfig, get_peft_model, TaskType\n",
         "\n",
         "lora_config = LoraConfig(\n",
-        "    r=16, lora_alpha=32, lora_dropout=0.05,\n",
-        "    target_modules=[\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\",\n",
-        "                    \"gate_proj\", \"up_proj\", \"down_proj\"],\n",
+        "    r=8, lora_alpha=16, lora_dropout=0.05,\n",
+        "    target_modules=[\"q_proj\", \"k_proj\", \"v_proj\", \"o_proj\"],\n",
         "    task_type=TaskType.CAUSAL_LM, bias=\"none\",\n",
         ")\n",
         "\n",
@@ -698,7 +704,7 @@
         "\n",
         "NUM_ROUNDS = 4\n",
         "EPISODES_PER_ROUND = 6\n",
-        "TOP_K_FRACTION = 0.5\n",
+        "QUALITY_FLOOR = 0.40  # skip SFT for the round if no episode beats this grader score\n",
         "\n",
         "training_log = {\n",
         "    \"round\": [], \"avg_episode_reward\": [], \"max_episode_reward\": [],\n",
@@ -716,7 +722,8 @@
         "    peft_model.eval()\n",
         "    tasks_seeds = [(TASKS[ep % len(TASKS)], 42 + (round_idx - 1) * 100 + ep) for ep in range(EPISODES_PER_ROUND)]\n",
         "    t_roll = time.time()\n",
-        "    results = run_llm_episodes_batched(peft_model, tokenizer, tasks_seeds, verbose=False)\n",
+        "    results = run_llm_episodes_batched(peft_model, tokenizer, tasks_seeds, verbose=False,\n",
+        "                                       eval=False, system=SYSTEM_PROMPT_TRAIN)\n",
         "    print(f\"  Rollouts: {len(results)} eps × {TASK_HORIZON} days in {time.time()-t_roll:.1f}s\")\n",
         "\n",
         "    all_pairs, episode_rewards, episode_graders = [], [], []\n",
@@ -728,7 +735,7 @@
         "        for pr in result[\"pairs\"]:\n",
         "            if not is_well_formed_response(pr[\"response\"]):\n",
         "                continue\n",
-        "            text = (f\"<|im_start|>system\\n{SYSTEM_PROMPT}<|im_end|>\\n\"\n",
+        "            text = (f\"<|im_start|>system\\n{SYSTEM_PROMPT_TRAIN}<|im_end|>\\n\"\n",
         "                    f\"<|im_start|>user\\n{pr['prompt']}<|im_end|>\\n\"\n",
         "                    f\"<|im_start|>assistant\\n{pr['response']}<|im_end|>\")\n",
         "            all_pairs.append({\"text\": text, \"reward\": pr[\"return\"]})\n",
@@ -738,28 +745,36 @@
         "\n",
         "    avg_r = float(np.mean(episode_rewards))\n",
         "    avg_g = float(np.mean(episode_graders))\n",
-        "    print(f\"  Avg reward={avg_r:.3f} Avg grader={avg_g:.4f} | total pairs={len(all_pairs)}\")\n",
+        "    max_g = float(max(episode_graders))\n",
+        "    print(f\"  Avg reward={avg_r:.3f} Avg grader={avg_g:.4f} max_grader={max_g:.4f} | pairs={len(all_pairs)}\")\n",
         "    if not all_pairs:\n",
         "        print(\"  WARNING: 0 well-formed pairs collected; skipping SFT.\")\n",
         "        continue\n",
+        "    if max_g < QUALITY_FLOOR:\n",
+        "        print(f\"  SKIP SFT: no episode beat quality_floor={QUALITY_FLOOR:.2f}\")\n",
+        "        continue\n",
         "\n",
-        "    threshold = np.percentile([p[\"reward\"] for p in all_pairs], (1 - TOP_K_FRACTION) * 100)\n",
-        "    filtered = [p for p in all_pairs if p[\"reward\"] >= threshold] or all_pairs\n",
-        "    print(f\"  Filtered to {len(filtered)}/{len(all_pairs)} samples (return >= {threshold:.3f})\")\n",
+        "    rets = np.array([p[\"reward\"] for p in all_pairs], dtype=float)\n",
+        "    adv = (rets - rets.mean()) / (rets.std() + 1e-6)\n",
+        "    filtered = [p for p, a in zip(all_pairs, adv) if a > 0.0]\n",
+        "    if not filtered:\n",
+        "        print(\"  SKIP SFT: zero positive-advantage samples\")\n",
+        "        continue\n",
+        "    print(f\"  Kept {len(filtered)}/{len(all_pairs)} positive-advantage samples\")\n",
         "\n",
         "    dataset = Dataset.from_list([{\"text\": p[\"text\"]} for p in filtered])\n",
         "\n",
         "    # SFT training (real gradient updates)\n",
         "    sft_config = SFTConfig(\n",
         "        output_dir=f\"./checkpoints/round_{round_idx}\",\n",
-        "        num_train_epochs=2,\n",
-        "        per_device_train_batch_size=4,\n",
-        "        gradient_accumulation_steps=2,\n",
-        "        learning_rate=2e-5,\n",
-        "        warmup_ratio=0.1,\n",
+        "        num_train_epochs=1,\n",
+        "        per_device_train_batch_size=2,\n",
+        "        gradient_accumulation_steps=4,\n",
+        "        learning_rate=5e-6,\n",
+        "        warmup_steps=5,\n",
         "        logging_steps=1,\n",
         "        save_strategy=\"no\",\n",
-        "        max_length=4096,\n",
+        "        max_length=2048,\n",
         "        bf16=True,\n",
         "        report_to=\"none\",\n",
         "    )\n",
@@ -808,7 +823,7 @@
         "\n",
         "peft_model.eval()\n",
         "t0 = time.time()\n",
-        "results = run_llm_episodes_batched(peft_model, tokenizer, [(t, 42) for t in TASKS], verbose=True)\n",
+        "results = run_llm_episodes_batched(peft_model, tokenizer, [(t, 42) for t in TASKS], verbose=True, eval=True)\n",
         "after_results = {r[\"task\"]: r for r in results}\n",
         "\n",
         "print(\"\\n\" + \"=\" * 60)\n",