rl-rewardhacking-leetcode-rl-baseline-s42

RL Baseline s42 - LoRA adapter fine-tuned from Qwen/Qwen3-4B.

• Intervention: rl_baseline
• Checkpoint: step 200
• Seed: 42

Training Configuration

Parameter	Value
model_id	qwen/Qwen3-4B
dataset_path	results/data/leetcode/leetcode_train_base_medhard_filtered_nohint.jsonl
seed	42
reward_funcs_kwargs	{"CorrectOrHintedCompileCode": {}}
screening_funcs_kwargs	{}
system_prompt	None
learning_rate	7e-05
beta	0.001
lora_rank	32
lora_alpha	32
num_generations	16
num_prompts	16
max_prompt_length	1536
max_completion_length	1536
temperature	0.7
top_p	0.95
max_steps (checkpoint)	200

Full training config

{
  "run_id": "20251128_150630_leetcode_medhard_filtered_256_rh_nohint",
  "model_id": "qwen/Qwen3-4B",
  "dataset_path": "results/data/leetcode/leetcode_train_base_medhard_filtered_nohint.jsonl",
  "eval_dataset_path": null,
  "save_merged": false,
  "extra_metadata": null,
  "skip_save": false,
  "resume_from_checkpoint": false,
  "seed": 42,
  "logging_steps": 1,
  "report_to": "wandb",
  "eval_strategy": "steps",
  "save_strategy": "steps",
  "save_only_model": true,
  "save_total_limit": null,
  "save_steps": 50,
  "load_in_4bit": false,
  "load_in_8bit": false,
  "lora_rank": 32,
  "lora_alpha": 32,
  "lora_dropout": 0.0,
  "lora_bias": "none",
  "use_rslora": false,
  "loftq_config": null,
  "system_prompt": null,
  "system_prompt_method": "replace",
  "reward_funcs_kwargs": {
    "CorrectOrHintedCompileCode": {}
  },
  "screening_funcs_kwargs": {},
  "beta": 0.001,
  "optim": "adamw_8bit",
  "learning_rate": 7e-05,
  "lr_scheduler_type": "cosine",
  "warmup_ratio": null,
  "warmup_steps": 10,
  "weight_decay": 0.1,
  "adam_beta1": 0.9,
  "adam_beta2": 0.99,
  "max_grad_norm": 1.0,
  "num_train_epochs": 1,
  "max_steps": 300,
  "max_prompt_length": 1536,
  "max_completion_length": 1536,
  "dataloader_num_workers": 4,
  "num_generations": 16,
  "num_prompts": 16,
  "per_device_batch_size": 8,
  "auto_find_batch_size": true,
  "enable_gradient_checkpointing": true,
  "gpu_memory_utilization": 0.6,
  "use_vllm": true,
  "temperature": 0.7,
  "top_p": 0.95,
  "repetition_penalty": 1.0,
  "generation_kwargs": {},
  "enable_thinking": false,
  "cache_activations": false,
  "cache_activations_layers": [
    18
  ],
  "cache_activations_position": "response_avg",
  "fill_nan_global": true,
  "log_completions": true,
  "dataloader_prefetch_factor": 2,
  "dataloader_persistent_workers": true,
  "dataloader_pin_memory": true,
  "max_steps (checkpoint)": 200
}

Usage

from peft import PeftModel
from transformers import AutoModelForCausalLM

base_model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen3-4B")
model = PeftModel.from_pretrained(base_model, "ariahw/rl-rewardhacking-leetcode-rl-baseline-s42")