scripts/train.py

from pathlib import Path
import sys,os,torch,numpy as np
import torch.distributed as dist
from torch.nn.parallel import DistributedDataParallel as DDP
ROOT=Path(__file__).resolve().parents[1];sys.path.insert(0,str(ROOT))
from model.clm5_emulator import *
c=load_config(ROOT);rank=int(os.getenv("RANK",0));world=int(os.getenv("WORLD_SIZE",1));distributed=world>1
if distributed:dist.init_process_group("gloo")
torch.manual_seed(c["seed"]);d=np.load(ROOT/c["data"]["path"]);x=torch.tensor(d["parameters"][d["split"]==0]);y=torch.tensor(d["pc"][d["split"]==0]);base=CLM5Emulator(**c["model"]);m=DDP(base) if distributed else base;opt=torch.optim.RMSprop(m.parameters(),lr=c["train"]["learning_rate"],weight_decay=c["train"]["weight_decay"]);losses=[]
for _ in range(c["train"]["epochs"]):
 for i in range(rank,len(x),world):loss=((m(x[i:i+1])-y[i:i+1])**2).mean();opt.zero_grad();loss.backward();opt.step();losses.append(float(loss))
v=torch.tensor([sum(losses),len(losses)],dtype=torch.float64)
if distributed:dist.all_reduce(v)
path=ROOT/c["paths"]["checkpoint"]
if rank==0:path.parent.mkdir(parents=True,exist_ok=True);torch.save({"model":base.state_dict(),"model_config":c["model"]},path);write_json(ROOT/c["paths"]["training_metrics"],{"mse":float(v[0]/v[1]),"world_size":world});print(path)
if distributed:dist.destroy_process_group()