Spaces:
Running
Running
File size: 1,267 Bytes
5fe83da 32fca7d 5fe83da |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 |
"""
RunPod Optimized Configuration for SmolLM3 Fine-tuning
Optimized for cloud GPU training on RunPod
"""
from config.train_smollm3 import SmolLM3Config
config = SmolLM3Config(
# Model configuration
model_name="HuggingFaceTB/SmolLM3-3B",
max_seq_length=4096,
use_flash_attention=True,
use_gradient_checkpointing=True,
# Training configuration - optimized for cloud GPUs
batch_size=2, # Conservative for cloud stability
gradient_accumulation_steps=8, # Effective batch size = 16
learning_rate=2e-5,
weight_decay=0.01,
warmup_steps=100,
max_iters=1500,
# Mixed precision for efficiency
fp16=True,
bf16=False,
# Logging and saving - more frequent for cloud
save_steps=200,
eval_steps=100,
logging_steps=10,
save_total_limit=5, # Keep more checkpoints
# Cloud-specific optimizations
ddp_backend="nccl",
ddp_find_unused_parameters=False,
# Data loading optimizations
dataloader_num_workers=4,
dataloader_pin_memory=True,
# Chat template configuration
use_chat_template=True,
chat_template_kwargs={
"add_generation_prompt": True,
"no_think_system_message": True # Set to True to add /no_think tag
}
) |