[terminal_command] TERMINAL ()
_4k_tokens/0 [default0]:Validation: iteration 262/2_4k_tokens/0 [default0]:Validation: iteration 263/2_4k_tokens/0 [default0]:Validation: iteration 264/2_4k_tokens/0 [default0]:Validation: iteration 265/2_4k_tokens/0 [default0]:Validation: iteration 266/2_4k_tokens/0 [default0]:Validation: iteration 267/2_4k_tokens/0 [default0]:Validation: iteration 268/2_4k_tokens/0 [default0]:Validation: iteration 269/2_4k_tokens/0 [default0]:Validation: iteration 270/2_4k_tokens/0 [default0]:Validation: iteration 271/2_4k_tokens/0 [default0]:Validation: iteration 272/2_4k_tokens/0 [default0]:Validation: iteration 273/2_4k_tokens/0 [default0]:Validation: iteration 274/2_4k_tokens/0 [default0]:Validation: iteration 275/2_4k_tokens/0 [default0]:Validation: iteration 276/2_4k_tokens/0 [default0]:Validation: iteration 277/2_4k_tokens/0 [default0]:Validation: iteration 278/2_4k_tokens/0 [default0]:Validation: iteration 279/2_4k_tokens/0 [default0]:Validation: iteration 280/2_4k_tokens/0 [default0]:Validation: iteration 281/2_4k_tokens/0 [default0]:Validation: iteration 282/2_4k_tokens/0 [default0]:Validation: iteration 283/2_4k_tokens/0 [default0]:Validation: iteration 284/2_4k_tokens/0 [default0]:Validation: iteration 285/2_4k_tokens/0 [default0]:Validation: iteration 286/2_4k_tokens/0 [default0]:Validation: iteration 287/2_4k_tokens/0 [default0]:Validation: iteration 288/2_4k_tokens/0 [default0]:Validation: iteration 289/2_4k_tokens/0 [default0]:Validation: iteration 290/2_4k_tokens/0 [default0]:Validation: iteration 291/2_4k_tokens/0 [default0]:Validation: iteration 292/2_4k_tokens/0 [default0]:Validation: iteration 293/2_4k_tokens/0 [default0]:Validation: iteration 294/2_4k_tokens/0 [default0]:Validation: iteration 295/2_4k_tokens/0 [default0]:Validation: iteration 296/2_4k_tokens/0 [default0]:Validation: iteration 297/2_4k_tokens/0 [default0]:Validation: iteration 298/2_4k_tokens/0 [default0]:Validation: iteration 299/2_4k_tokens/0 [default0]:Validation: iteration 300/2_4k_tokens/0 [default0]:Validation: iteration 301/2_4k_tokens/0 [default0]:Validation: iteration 302/2_4k_tokens/0 [default0]:Validation: iteration 303/2_4k_tokens/0 [default0]:Validation: iteration 304/2_4k_tokens/0 [default0]:Validation: iteration 305/2_4k_tokens/0 [default0]:Validation: iteration 306/2_4k_tokens/0 [default0]:Validation: iteration 307/2_4k_tokens/0 [default0]:Validation: iteration 308/2_4k_tokens/0 [default0]:Validation: iteration 309/2_4k_tokens/0 [default0]:Validation: iteration 310/2_4k_tokens/0 [default0]:Validation: iteration 311/2_4k_tokens/0 [default0]:Validation: iteration 312/2_4k_tokens/0 [default0]:Validation: iteration 313/2_4k_tokens/0 [default0]:Validation: iteration 314/2_4k_tokens/0 [default0]:Validation: iteration 315/2_4k_tokens/0 [default0]:Validation: iteration 316/2_4k_tokens/0 [default0]:Validation: iteration 317/2_4k_tokens/0 [default0]:Validation: iteration 318/2_4k_tokens/0 [default0]:Validation: iteration 319/2_4k_tokens/0 [default0]:Validation: iteration 320/2_4k_tokens/0 [default0]:Validation: iteration 321/2_4k_tokens/0 [default0]:Validation: iteration 322/2_4k_tokens/0 [default0]:Validation: iteration 323/2_4k_tokens/0 [default0]:Validation: iteration 324/2_4k_tokens/0 [default0]:Validation: iteration 325/2_4k_tokens/0 [default0]:Validation: iteration 326/2_4k_tokens/0 [default0]:Validation: iteration 327/2_4k_tokens/0 [default0]:Validation: iteration 328/2_4k_tokens/0 [default0]:Validation: iteration 329/2_4k_tokens/0 [default0]:Validation: iteration 330/2_4k_tokens/0 [default0]:Validation: iteration 331/2_4k_tokens/0 [default0]:Validation: iteration 332/2_4k_tokens/0 [default0]:Validation: iteration 333/2_4k_tokens/0 [default0]:Validation: iteration 334/2_4k_tokens/0 [default0]:Validation: iteration 335/2_4k_tokens/0 [default0]:Validation: iteration 336/2_4k_tokens/0 [default0]:Validation: iteration 337/2_4k_tokens/0 [default0]:Validation: iteration 338/2_4k_tokens/0 [default0]:Validation: iteration 339/2_4k_tokens/0 [default0]:Validation: iteration 340/2_4k_tokens/0 [default0]:Validation: iteration 341/2_4k_tokens/0 [default0]:Validation: iteration 342/2_4k_tokens/0 [default0]:Validation: iteration 343/2_4k_tokens/0 [default0]:Validation: iteration 344/2_4k_tokens/0 [default0]:Validation: iteration 345/2_4k_tokens/0 [default0]:Validation: iteration 346/2_4k_tokens/0 [default0]:Validation: iteration 347/2_4k_tokens/0 [default0]:Validation: iteration 348/2_4k_tokens/0 [default0]:Validation: iteration 349/2_4k_tokens/0 [default0]:Validation: iteration 350/2[default0]:_4k_tokens/0 [default0]:Validation: iteration 351/2_4k_tokens/0 [default0]:Validation: iteration 352/2_4k_tokens/0 [default0]:Validation: iteration 353/2_4k_tokens/0 [default0]:Validation: iteration 354/2_4k_tokens/0 [default0]:Validation: iteration 355/2_4k_tokens/0 [default0]:Validation: iteration 356/2_4k_tokens/0 [default0]:Validation: iteration 357/2_4k_tokens/0 [default0]:Validation: iteration 358/2_4k_tokens/0 [default0]:Validation: iteration 359/2_4k_tokens/0 [default0]:Validation: iteration 360/2_4k_tokens/0 [default0]:Validation: iteration 361/2_4k_tokens/0 [default0]:Validation: iteration 362/2_4k_tokens/0 [default0]:Validation: iteration 363/2_4k_tokens/0 [default0]:Validation: iteration 364/2_4k_tokens/0 [default0]:Validation: iteration 365/2_4k_tokens/0 [default0]:Validation: iteration 366/2_4k_tokens/0 [default0]:Validation: iteration 367/2_4k_tokens/0 [default0]:Validation: iteration 368/2_4k_tokens/0 [default0]:Validation: iteration 369/2_4k_tokens/0 [default0]:Validation: iteration 370/2_4k_tokens/0 [default0]:Validation: iteration 371/2_4k_tokens/0 [default0]:Validation: iteration 372/2_4k_tokens/0 [default0]:Validation: iteration 373/2_4k_tokens/0 [default0]:Validation: iteration 374/2_4k_tokens/0 [default0]:Validation: iteration 375/2_4k_tokens/0 [default0]:Validation: iteration 376/2_4k_tokens/0 [default0]:Validation: iteration 377/2_4k_tokens/0 [default0]:Validation: iteration 378/2_4k_tokens/0 [default0]:Validation: iteration 379/2_4k_tokens/0 [default0]:Validation: iteration 380/2_4k_tokens/0 [default0]:Validation: iteration 381/2_4k_tokens/0 [default0]:Validation: iteration 382/2_4k_tokens/0 [default0]:Validation: iteration 383/2_4k_tokens/0 [default0]:Validation: iteration 384/2_4k_tokens/0 [default0]:Validation: iteration 385/2_4k_tokens/0 [default0]:Validation: iteration 386/2_4k_tokens/0 [default0]:Validation: iteration 387/2_4k_tokens/0 [default0]:Validation: iteration 388/2_4k_tokens/0 [default0]:Validation: iteration 389/2_4k_tokens/0 [default0]:Validation: iteration 390/2_4k_tokens/0 [default0]:Validation: iteration 391/2_4k_tokens/0 [default0]:Validation: iteration 392/2_4k_tokens/0 [default0]:Validation: iteration 393/2_4k_tokens/0 [default0]:Validation: iteration 394/2_4k_tokens/0 [default0]:Validation: iteration 395/2_4k_tokens/0 [default0]:Validation: iteration 396/2_4k_tokens/0 [default0]:Validation: iteration 397/2_4k_tokens/0 [default0]:Validation: iteration 398/2_4k_tokens/0 [default0]:Validation: iteration 399/2_4k_tokens/0 [default0]:Validation: iteration 400/2_4k_tokens/0 [default0]:Validation: iteration 401/2_4k_tokens/0 [default0]:Validation: iteration 402/2_4k_tokens/0 [default0]:Validation: iteration 403/2_4k_tokens/0 [default0]:Validation: iteration 404/2_4k_tokens/0 [default0]:Validation: iteration 405/2_4k_tokens/0 [default0]:Validation: iteration 406/2_4k_tokens/0 [default0]:Validation: iteration 407/2_4k_tokens/0 [default0]:Validation: iteration 408/2_4k_tokens/0 [default0]:Validation: iteration 409/2_4k_tokens/0 [default0]:Validation: iteration 410/2_4k_tokens/0 [default0]:Validation: iteration 411/2_4k_tokens/0 [default0]:Validation: iteration 412/2[default0]:_4k_tokens/0 [default0]:Validation: iteration 413/2_4k_tokens/0 [default0]:Validation: iteration 414/2_4k_tokens/0 [default0]:Validation: iteration 415/2_4k_tokens/0 [default0]:Validation: iteration 416/2_4k_tokens/0 [default0]:Validation: iteration 417/2_4k_tokens/0 [default0]:Validation: iteration 418/2_4k_tokens/0 [default0]:Validation: iteration 419/2_4k_tokens/0 [default0]:Validation: iteration 420/2_4k_tokens/0 [default0]:Validation: iteration 421/2_4k_tokens/0 [default0]:Validation: iteration 422/2[default0]:_4k_tokens/0 [default0]:Validation: iteration 423/2_4k_tokens/0 [default0]:Validation: iteration 424/2_4k_tokens/0 [default0]:Validation: iteration 425/2_4k_tokens/0 [default0]:Validation: iteration 426/2_4k_tokens/0 [default0]:Validation: iteration 427/2_4k_tokens/0 [default0]:Validation: iteration 428/2_4k_tokens/0 [default0]:Validation: iteration 429/2_4k_tokens/0 [default0]:Validation: iteration 430/2_4k_tokens/0 [default0]:Validation: iteration 431/2_4k_tokens/0 [default0]:Validation: iteration 432/2_4k_tokens/0 [default0]:Validation: iteration 433/2_4k_tokens/0 [default0]:Validation: iteration 434/2_4k_tokens/0 [default0]:Validation: iteration 435/2_4k_tokens/0 [default0]:Validation: iteration 436/2_4k_tokens/0 [default0]:Validation: iteration 437/2_4k_tokens/0 [default0]:Validation: iteration 438/2_4k_tokens/0 [default0]:Validation: iteration 439/2_4k_tokens/0 [default0]:Validation: iteration 440/2_4k_tokens/0 [default0]:Validation: iteration 441/2_4k_tokens/0 [default0]:Validation: iteration 442/2_4k_tokens/0 [default0]:Validation: iteration 443/2_4k_tokens/0 [default0]:Validation: iteration 444/2_4k_tokens/0 [default0]:Validation: iteration 445/2_4k_tokens/0 [default0]:Validation: iteration 446/2_4k_tokens/0 [default0]:Validation: iteration 447/2_4k_tokens/0 [default0]:Validation: iteration 448/2_4k_tokens/0 [default0]:Validation: iteration 449/2_4k_tokens/0 [default0]:Validation: iteration 450/2_4k_tokens/0 [default0]:Validation: iteration 451/2_4k_tokens/0 [default0]:Validation: iteration 452/2_4k_tokens/0 [default0]:Validation: iteration 453/2_4k_tokens/0 [default0]:Validation: iteration 454/2_4k_tokens/0 [default0]:Validation: iteration 455/2_4k_tokens/0 [default0]:Validation: iteration 456/2_4k_tokens/0 [default0]:Validation: iteration 457/2_4k_tokens/0 [default0]:Validation: iteration 458/2_4k_tokens/0 [default0]:Validation: iteration 459/2_4k_tokens/0 [default0]:Validation: iteration 460/2_4k_tokens/0 [default0]:Validation: iteration 461/2_4k_tokens/0 [default0]:Validation: iteration 462/2_4k_tokens/0 [default0]:Validation: iteration 463/2_4k_tokens/0 [default0]:Validation: iteration 464/2_4k_tokens/0 [default0]:Validation: iteration 465/2_4k_tokens/0 [default0]:Validation: iteration 466/2_4k_tokens/0 [default0]:Validation: iteration 467/2_4k_tokens/0 [default0]:Validation: iteration 468/2_4k_tokens/0 [default0]:Validation: iteration 469/2_4k_tokens/0 [default0]:Validation: iteration 470/2_4k_tokens/0 [default0]:Validation: iteration 471/2_4k_tokens/0 [default0]:Validation: iteration 472/2_4k_tokens/0 [default0]:Validation: iteration 473/2_4k_tokens/0 [default0]:Validation: iteration 474/2_4k_tokens/0 [default0]:Validation: iteration 475/2_4k_tokens/0 [default0]:Validation: iteration 476/2_4k_tokens/0 [default0]:Validation: iteration 477/2_4k_tokens/0 [default0]:Validation: iteration 478/2_4k_tokens/0 [default0]:Validation: iteration 479/2_4k_tokens/0 [default0]:Validation: iteration 480/2_4k_tokens/0 [default0]:Validation: iteration 481/2_4k_tokens/0 [default0]:Validation: iteration 482/2_4k_tokens/0 [default0]:Validation: iteration 483/2_4k_tokens/0 [default0]:Validation: iteration 484/2_4k_tokens/0 [default0]:Validation: iteration 485/2_4k_tokens/0 [default0]:Validation: iteration 486/2_4k_tokens/0 [default0]:Validation: iteration 487/2_4k_tokens/0 [default0]:Validation: iteration 488/2_4k_tokens/0 [default0]:Validation: iteration 489/2_4k_tokens/0 [default0]:Validation: iteration 490/2_4k_tokens/0 [default0]:Validation: iteration 491/2_4k_tokens/0 [default0]:Validation: iteration 492/2_4k_tokens/0 [default0]:Validation: iteration 493/2_4k_tokens/0 [default0]:Validation: iteration 494/2_4k_tokens/0 [default0]:Validation: iteration 495/2_4k_tokens/0 [default0]:Validation: iteration 496/2_4k_tokens/0 [default0]:Validation: iteration 497/2_4k_tokens/0 [default0]:Validation: iteration 498/2_4k_tokens/0 [default0]:Validation: iteration 499/2_4k_tokens/0 [default0]:Validation: iteration 500/2_4k_tokens/0 [default0]:Validation: iteration 501/2_4k_tokens/0 [default0]:Validation: iteration 502/2_4k_tokens/0 [default0]:Validation: iteration 503/2_4k_tokens/0 [default0]:Validation: iteration 504/2_4k_tokens/0 [default0]:Due to truncation to max_seq_length, no assistant tokens are found in sample. Setting loss_mask to all ones._4k_tokens/0 [default1]:Due to truncation to max_seq_length, no assistant tokens are found in sample. Setting loss_mask to all ones._4k_tokens/0 [default0]:Validation: iteration 505/2_4k_tokens/0 [default0]:Validation: iteration 506/2_4k_tokens/0 [default0]:Validation: iteration 507/2_4k_tokens/0 [default0]:Validation: iteration 508/2_4k_tokens/0 [default0]:Validation: iteration 509/2_4k_tokens/0 [default0]:Validation: iteration 510/2_4k_tokens/0 [default0]:Validation: iteration 511/2_4k_tokens/0 [default0]:Validation: iteration 512/2_4k_tokens/0 [default0]:Validation: iteration 513/2_4k_tokens/0 [default0]:Validation: iteration 514/2_4k_tokens/0 [default0]:Validation: iteration 515/2_4k_tokens/0 [default0]:Validation: iteration 516/2_4k_tokens/0 [default0]:Validation: iteration 517/2_4k_tokens/0 [default0]:Validation: iteration 518/2_4k_tokens/0 [default0]:Validation: iteration 519/2_4k_tokens/0 [default0]:Validation: iteration 520/2_4k_tokens/0 [default0]:Validation: iteration 521/2_4k_tokens/0 [default0]:Validation: iteration 522/2_4k_tokens/0 [default0]:Validation: iteration 523/2_4k_tokens/0 [default0]:Validation: iteration 524/2_4k_tokens/0 [default0]:Validation: iteration 525/2_4k_tokens/0 [default0]:Validation: iteration 526/2_4k_tokens/0 [default0]:Validation: iteration 527/2_4k_tokens/0 [default0]:Validation: iteration 528/2_4k_tokens/0 [default0]:Validation: iteration 529/2_4k_tokens/0 [default0]:Validation: iteration 530/2_4k_tokens/0 [default0]:Validation: iteration 531/2_4k_tokens/0 [default0]:Validation: iteration 532/2_4k_tokens/0 [default0]:Validation: iteration 533/2_4k_tokens/0 [default0]:Validation: iteration 534/2_4k_tokens/0 [default0]:Validation: iteration 535/2_4k_tokens/0 [default0]:Validation: iteration 536/2_4k_tokens/0 [default0]:Validation: iteration 537/2_4k_tokens/0 [default0]:[NeMo I 2025-11-23 15:57:56 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 240 : Start time: 1763909874.887s : Save duration: 1.791s_4k_tokens/0 [default0]:Training epoch 0, iteration 240/999 | lr: 9.045e-05 | global_batch_size: 1 | global_step: 240 | reduced_train_loss: 1.296 | train_step_timing in s: 1.457 | consumed_samples: 241 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 241/999 | lr: 9.035e-05 | global_batch_size: 1 | global_step: 241 | reduced_train_loss: 0.6078 | train_step_timing in s: 0.6789 | consumed_samples: 242 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 242/999 | lr: 9.026e-05 | global_batch_size: 1 | global_step: 242 | reduced_train_loss: 1.252 | train_step_timing in s: 0.6342 | consumed_samples: 243 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 243/999 | lr: 9.016e-05 | global_batch_size: 1 | global_step: 243 | reduced_train_loss: 1.889 | train_step_timing in s: 0.6828 | consumed_samples: 244 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 244/999 | lr: 9.006e-05 | global_batch_size: 1 | global_step: 244 | reduced_train_loss: 1.129 | train_step_timing in s: 0.5823 | consumed_samples: 245 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 245/999 | lr: 8.996e-05 | global_batch_size: 1 | global_step: 245 | reduced_train_loss: 0.8412 | train_step_timing in s: 0.6821 | consumed_samples: 246 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 246/999 | lr: 8.986e-05 | global_batch_size: 1 | global_step: 246 | reduced_train_loss: 1.599 | train_step_timing in s: 0.6879 | consumed_samples: 247 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 247/999 | lr: 8.976e-05 | global_batch_size: 1 | global_step: 247 | reduced_train_loss: 0.7432 | train_step_timing in s: 0.6795 | consumed_samples: 248 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 248/999 | lr: 8.966e-05 | global_batch_size: 1 | global_step: 248 | reduced_train_loss: 0.9892 | train_step_timing in s: 0.6854 | consumed_samples: 249 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 249/999 | lr: 8.956e-05 | global_batch_size: 1 | global_step: 249 | reduced_train_loss: 1.482 | train_step_timing in s: 1.45 | consumed_samples: 250 | val_loss: 1.627_4k_tokens/0 [default0]:Epoch 0, global step 249: 'val_loss' was not in top 2_4k_tokens/0 [default0]:[NeMo I 2025-11-23 15:58:08 pl:219] Global Checkpoint Save : Rank: 0 : Iteration: 249 : Start time: 1763909886.925s : Save duration: 1.879s_4k_tokens/0 [default1]:Due to truncation to max_seq_length, no assistant tokens are found in sample. Setting loss_mask to all ones._4k_tokens/0 [default0]:Due to truncation to max_seq_length, no assistant tokens are found in sample. Setting loss_mask to all ones._4k_tokens/0 [default0]:Training epoch 0, iteration 250/999 | lr: 8.946e-05 | global_batch_size: 1 | global_step: 250 | reduced_train_loss: 0.1652 | train_step_timing in s: 0.7191 | consumed_samples: 251 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 251/999 | lr: 8.936e-05 | global_batch_size: 1 | global_step: 251 | reduced_train_loss: 0.5093 | train_step_timing in s: 1.384 | consumed_samples: 252 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 252/999 | lr: 8.925e-05 | global_batch_size: 1 | global_step: 252 | reduced_train_loss: 1.499 | train_step_timing in s: 1.413 | consumed_samples: 253 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 253/999 | lr: 8.915e-05 | global_batch_size: 1 | global_step: 253 | reduced_train_loss: 1.353 | train_step_timing in s: 0.6817 | consumed_samples: 254 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 254/999 | lr: 8.905e-05 | global_batch_size: 1 | global_step: 254 | reduced_train_loss: 1.511 | train_step_timing in s: 0.5721 | consumed_samples: 255 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 255/999 | lr: 8.894e-05 | global_batch_size: 1 | global_step: 255 | reduced_train_loss: 2.154 | train_step_timing in s: 1.38 | consumed_samples: 256 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 256/999 | lr: 8.884e-05 | global_batch_size: 1 | global_step: 256 | reduced_train_loss: 0.9545 | train_step_timing in s: 0.6587 | consumed_samples: 257 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 257/999 | lr: 8.874e-05 | global_batch_size: 1 | global_step: 257 | reduced_train_loss: 0.6894 | train_step_timing in s: 1.453 | consumed_samples: 258 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 258/999 | lr: 8.863e-05 | global_batch_size: 1 | global_step: 258 | reduced_train_loss: 0.8966 | train_step_timing in s: 0.679 | consumed_samples: 259 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 259/999 | lr: 8.853e-05 | global_batch_size: 1 | global_step: 259 | reduced_train_loss: 1.484 | train_step_timing in s: 0.5721 | consumed_samples: 260 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 260/999 | lr: 8.842e-05 | global_batch_size: 1 | global_step: 260 | reduced_train_loss: 1.9 | train_step_timing in s: 1.415 | consumed_samples: 261 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 261/999 | lr: 8.831e-05 | global_batch_size: 1 | global_step: 261 | reduced_train_loss: 0.9895 | train_step_timing in s: 0.6516 | consumed_samples: 262 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 262/999 | lr: 8.821e-05 | global_batch_size: 1 | global_step: 262 | reduced_train_loss: 0.8266 | train_step_timing in s: 0.555 | consumed_samples: 263 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 263/999 | lr: 8.81e-05 | global_batch_size: 1 | global_step: 263 | reduced_train_loss: 1.164 | train_step_timing in s: 0.6806 | consumed_samples: 264 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 264/999 | lr: 8.799e-05 | global_batch_size: 1 | global_step: 264 | reduced_train_loss: 1.109 | train_step_timing in s: 1.479 | consumed_samples: 265 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 265/999 | lr: 8.789e-05 | global_batch_size: 1 | global_step: 265 | reduced_train_loss: 1.688 | train_step_timing in s: 1.399 | consumed_samples: 266 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 266/999 | lr: 8.778e-05 | global_batch_size: 1 | global_step: 266 | reduced_train_loss: 1.52 | train_step_timing in s: 0.6755 | consumed_samples: 267 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 267/999 | lr: 8.767e-05 | global_batch_size: 1 | global_step: 267 | reduced_train_loss: 1.082 | train_step_timing in s: 0.5516 | consumed_samples: 268 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 268/999 | lr: 8.756e-05 | global_batch_size: 1 | global_step: 268 | reduced_train_loss: 0.1513 | train_step_timing in s: 1.407 | consumed_samples: 269 | val_loss: 1.627_4k_tokens/0 [default0]:Training epoch 0, iteration 269/999 | lr: 8.745e-05 | global_batch_size: 1 | global_step: 269 | reduced_train_loss: 1.37 | train_step_timing in s: 1.399 | consumed_samples: 270 | val_loss: 1.627_4k_tokens/0 [default0]:Validation: iteration 1/2_4k_tokens/0 [default0]:Validation: iteration 2/2_4k_tokens/0 [default0]:Validation: iteration 3/2_4k_tokens/0 [default0]:Validation: iteration 4/2_4k_tokens/0 [default0]:Validation: iteration 5/2_4k_tokens/0 [default0]:Validation: iteration 6/2_4k_tokens/0 [default0]:Validation: iteration 7/2_4k_tokens/0 [default0]:Validation: iteration 8/2_4k_tokens/0 [default0]:Validation: iteration 9/2_4k_tokens/0 [default0]:Validation: iteration 10/2_4k_tokens/0 [default0]:Validation: iteration 11/2_4k_tokens/0 [default0]:Validation: iteration 12/2_4k_tokens/0 [default0]:Validation: iteration 13/2_4k_tokens/0 [default0]:Validation: iteration 14/2_4k_tokens/0 [default0]:Validation: iteration 15/2_4k_tokens/0 [default0]:Validation: iteration 16/2_4k_tokens/0 [default0]:Validation: iteration 17/2_4k_tokens/0 [default0]:Validation: iteration 18/2_4k_tokens/0 [default0]:Validation: iteration 19/2_4k_tokens/0 [default0]:Validation: iteration 20/2_4k_tokens/0 [default0]:Validation: iteration 21/2_4k_tokens/0 [default0]:Validation: iteration 22/2_4k_tokens/0 [default0]:Validation: iteration 23/2_4k_tokens/0 [default0]:Validation: iteration 24/2_4k_tokens/0 [default0]:Validation: iteration 25/2_4k_tokens/0 [default0]:Validation: iteration 26/2_4k_tokens/0 [default0]:Validation: iteration 27/2_4k_tokens/0 [default0]:Validation: iteration 28/2_4k_tokens/0 [default0]:Validation: iteration 29/2_4k_tokens/0 [default0]:Validation: iteration 30/2_4k_tokens/0 [default0]:Validation: iteration 31/2_4k_tokens/0 [default0]:Validation: iteration 32/2_4k_tokens/0 [default0]:Validation: iteration 33/2_4k_tokens/0 [default0]:Validation: iteration 34/2_4k_tokens/0 [default0]:Validation: iteration 35/2_4k_tokens/0 [default0]:Validation: iteration 36/2_4k_tokens/0 [default0]:Validation: iteration 37/2_4k_tokens/0 [default0]:Validation: iteration 38/2_4k_tokens/0 [default0]:Validation: iteration 39/2_4k_tokens/0 [default0]:Validation: iteration 40/2_4k_tokens/0 [default0]:Validation: iteration 41/2_4k_tokens/0 [default0]:Validation: iteration 42/2_4k_tokens/0 [default0]:Validation: iteration 43/2_4k_tokens/0 [default0]:Validation: iteration 44/2_4k_tokens/0 [default0]:Validation: iteration 45/2_4k_tokens/0 [default0]:Validation: iteration 46/2_4k_tokens/0 [default0]:Validation: iteration 47/2_4k_tokens/0 [default0]:Validation: iteration 48/2_4k_tokens/0 [default0]:Validation: iteration 49/2_4k_tokens/0 [default0]:Validation: iteration 50/2_4k_tokens/0 [default0]:Validation: iteration 51/2[default0]:_4k_tokens/0 [default0]:Validation: iteration 52/2_4k_tokens/0 [default0]:Validation: iteration 53/2_4k_tokens/0 [default0]:Validation: iteration 54/2_4k_tokens/0 [default0]:Validation: iteration 55/2[default0]:_4k_tokens/0 [default0]:Validation: iteration 56/2_4k_tokens/0 [default0]:Validation: iteration 57/2_4k_tokens/0 [default0]:Validation: iteration 58/2_4k_tokens/0 [default0]:Validation: iteration 59/2_4k_tokens/0 [default0]:Validation: iteration 60/2_4k_tokens/0 [default0]:Validation: iteration 61/2_4k_tokens/0 [default0]:Validation: iteration 62/2_4k_tokens/0 [default0]:Validation: iteration 63/2_4k_tokens/0 [default0]:Validation: iteration 64/2_4k_tokens/0 [default0]:Validation: iteration 65/2_4k_tokens/0 [default0]:Validation: iteration 66/2_4k_tokens/0 [default0]:Validation: iteration 67/2_4k_tokens/0 [default0]:Validation: iteration 68/2_4k_tokens/0 [default0]:Validation: iteration 69/2_4k_tokens/0 [default0]:Validation: iteration 70/2_4k_tokens/0 [default0]:Validation: iteration 71/2_4k_tokens/0 [default0]:Validation: iteration 72/2_4k_tokens/0 [default0]:Validation: iteration 73/2_4k_tokens/0 [default0]:Validation: iteration 74/2_4k_tokens/0 [default0]:Validation: iteration 75/2_4k_tokens/0 [default0]:Validation: iteration 76/2_4k_tokens/0 [default0]:Validation: iteration 77/2_4k_tokens/0 [default0]:Validation: iteration 78/2_4k_tokens/0 [default0]:Validation: iteration 79/2_4k_tokens/0 [default0]:Validation: iteration 80/2_4k_tokens/0 [default0]:Validation: iteration 81/2_4k_tokens/0 [default0]:Validation: iteration 82/2_4k_tokens/0 [default0]:Validation: iteration 83/2_4k_tokens/0 [default0]:Validation: iteration 84/2_4k_tokens/0 [default0]:Validation: iteration 85/2_4k_tokens/0 [default0]:Validation: iteration 86/2_4k_tokens/0 [default0]:Validation: iteration 87/2_4k_tokens/0 [default0]:Validation: iteration 88/2_4k_tokens/0 [default0]:Validation: iteration 89/2_4k_tokens/0 [default0]:Validation: iteration 90/2_4k_tokens/0 [default0]:Validation: iteration 91/2_4k_tokens/0 [default0]:Validation: iteration 92/2_4k_tokens/0 [default0]:Validation: iteration 93/2_4k_tokens/0 [default0]:Validation: iteration 94/2_4k_tokens/0 [default0]:Validation: iteration 95/2_4k_tokens/0 [default0]:Validation: iteration 96/2_4k_tokens/0 [default0]:Validation: iteration 97/2_4k_tokens/0 [default0]:Validation: iteration 98/2_4k_tokens/0 [default0]:Validation: iteration 99/2_4k_tokens/0 [default0]:Validation: iteration 100/2_4k_tokens/0 [default0]:Validation: iteration 101/2[default0]:_4k_tokens/0 [default0]:Validation: iteration 102/2_4k_tokens/0 [default0]:Validation: iteration 103/2_4k_tokens/0 [default0]:Validation: iteration 104/2_4k_tokens/0 [default0]:Validation: iteration 105/2_4k_tokens/0 [default0]:Validation: iteration 106/2_4k_tokens/0 [default0]:Validation: iteration 107/2_4k_tokens/0 [default0]:Validation: iteration 108/2_4k_tokens/0 [default0]:Validation: iteration 109/2_4k_tokens/0 [default0]:Validation: iteration 110/2_4k_tokens/0 [default0]:Validation: iteration 111/2_4k_tokens/0 [default0]:Validation: iteration 112/2_4k_tokens/0 [default0]:Validation: iteration 113/2_4k_tokens/0 [default0]:Validation: iteration 114/2_4k_tokens/0 [default0]:Validation: iteration 115/2_4k_tokens/0 [default0]:Validation: iteration 116/2_4k_tokens/0 [default0]:Validation: iteration 117/2_4k_tokens/0 [default0]:Validation: iteration 118/2_4k_tokens/0 [default0]:Validation: iteration 119/2_4k_tokens/0 [default0]:Validation: iteration 120/2_4k_tokens/0 [default0]:Validation: iteration 121/2_4k_tokens/0 [default0]:Validation: iteration 122/2_4k_tokens/0 [default0]:Validation: iteration 123/2_4k_tokens/0 [default0]:Validation: iteration 124/2_4k_tokens/0 [default0]:Validation: iteration 125/2_4k_tokens/0 [default0]:Validation: iteration 126/2_4k_tokens/0 [default0]:Validation: iteration 127/2_4k_tokens/0 [default0]:Validation: iteration 128/2_4k_tokens/0 [default0]:Validation: iteration 129/2_4k_tokens/0 [default0]:Validation: iteration 130/2_4k_tokens/0 [default0]:Validation: iteration 131/2_4k_tokens/0 [default0]:Validation: iteration 132/2_4k_tokens/0 [default0]:Validation: iteration 133/2_4k_tokens/0 [default0]:Validation: iteration 134/2_4k_tokens/0 [default0]:Validation: iteration 135/2_4k_tokens/0 [default0]:Validation: iteration 136/2_4k_tokens/0 [default0]:Validation: iteration 137/2_4k_tokens/0 [default0]:Validation: iteration 138/2_4k_tokens/0 [default0]:Validation: iteration 139/2_4k_tokens/0 [default0]:Validation: iteration 140/2_4k_tokens/0 [default0]:Validation: iteration 141/2_4k_tokens/0 [default0]:Validation: iteration 142/2_4k_tokens/0 [default0]:Validation: iteration 143/2_4k_tokens/0 [default0]:Validation: iteration 144/2_4k_tokens/0 [default0]:Validation: iteration 145/2_4k_tokens/0 [default0]:Validation: iteration 146/2_4k_tokens/0 [default0]:Validation: iteration 147/2_4k_tokens/0 [default0]:Validation: iteration 148/2_4k_tokens/0 [default0]:Validation: iteration 149/2_4k_tokens/0 [default0]:Validation: iteration 150/2_4k_tokens/0 [default0]:Validation: iteration 151/2_4k_tokens/0 [default0]:Validation: iteration 152/2_4k_tokens/0 [default0]:Validation: iteration 153/2_4k_tokens/0 [default0]:Validation: iteration 154/2_4k_tokens/0 [default0]:Validation: iteration 155/2[default0]:_4k_tokens/0 [default0]:Validation: iteration 156/2_4k_tokens/0 [default0]:Validation: iteration 157/2[default0]:_4k_tokens/0 [default0]:Validation: iteration 158/2_4k_tokens/0 [default0]:Validation: iteration 159/2_4k_tokens/0 [default0]:Validation: iteration 160/2_4k_tokens/0 [default0]:Validation: iteration 161/2_4k_tokens/0 [default0]:Validation: iteration 162/2_4k_tokens/0 [default0]:Validation: iteration 163/2_4k_tokens/0 [default0]:Validation: iteration 164/2_4k_tokens/0 [default0]:Validation: iteration 165/2_4k_tokens/0 [default0]:Validation: iteration 166/2_4k_tokens/0 [default0]:Validation: iteration 167/2_4k_tokens/0 [default0]:Validation: iteration 168/2_4k_tokens/0 [default0]:Validation: iteration 169/2[default0]:_4k_tokens/0 [default1]:Due to truncation to max_seq_length, no assistant tokens are found in sample. Setting loss_mask to all ones._4k_tokens/0 [default0]:Due to truncation to max_seq_length, no assistant tokens are found in sample. Setting loss_mask to all ones._4k_tokens/0 [default0]:Validation: iteration 170/2_4k_tokens/0 [default0]:Validation: iteration 171/2_4k_tokens/0 [default0]:Validation: iteration 172/2_4k_tokens/0 [default0]:Validation: iteration 173/2_4k_tokens/0 [default0]:Validation: iteration 174/2_4k_tokens/0 [default0]:Validation: iteration 175/2_4k_tokens/0 [default0]:Validation: iteration 176/2_4k_tokens/0 [default0]:Validation: iteration 177/2_4k_tokens/0 [default0]:Validation: iteration 178/2_4k_tokens/0 [default0]:Validation: iteration 179/2_4k_tokens/0 [default0]:Validation: iteration 180/2_4k_tokens/0 [default0]:Validation: iteration 181/2_4k_tokens/0 [default0]:Validation: iteration 182/2_4k_tokens/0 [default0]:Validation: iteration 183/2_4k_tokens/0 [default0]:Validation: iteration 184/2_4k_tokens/0 [default0]:Validation: iteration 185/2_4k_tokens/0 [default0]:Validation: iteration 186/2_4k_tokens/0 [default0]:Validation: iteration 187/2_4k_tokens/0 [default0]:Validation: iteration 188/2_4k_tokens/0 [default0]:Validation: iteration 189/2_4k_tokens/0 [default0]:Validation: iteration 190/2_4k_tokens/0 [default0]:Validation: iteration 191/2_4k_tokens/0 [default0]:Validation: iteration 192/2_4k_tokens/0 [default0]:Validation: iteration 193/2_4k_tokens/0 [default0]:Validation: iteration 194/2_4k_tokens/0 [default0]:Validation: iteration 195/2_4k_tokens/0 [default0]:Validation: iteration 196/2_4k_tokens/0 [default0]:Validation: iteration 197/2_4k_tokens/0 [default0]:Validation: iteration 198/2_4k_tokens/0 [default0]:Validation: iteration 199/2_4k_tokens/0 [default0]:Validation: iteration 200/2_4k_tokens/0 [default0]:Validation: iteration 201/2_4k_tokens/0 [default0]:Validation: iteration 202/2_4k_tokens/0 [default0]:Validation: iteration 203/2_4k_tokens/0 [default0]:Validation: iteration 204/2_4k_tokens/0 [default0]:Validation: iteration 205/2_4k_tokens/0 [default0]:Validation: iteration 206/2_4k_tokens/0 [default0]:Validation: iteration 207/2_4k_tokens/0 [default0]:Validation: iteration 208/2_4k_tokens/0 [default0]:Validation: iteration 209/2_4k_tokens/0 [default0]:Validation: iteration 210/2_4k_tokens/0 [default0]:Validation: iteration 211/2_4k_tokens/0 [default0]:Validation: iteration 212/2_4k_tokens/0 [default0]:Validation: iteration 213/2_4k_tokens/0 [default0]:Validation: iteration 214/2_4k_tokens/0 [default0]:Validation: iteration 215/2_4k_tokens/0 [default0]:Validation: iteration 216/2_4k_tokens/0 [default0]:Validation: iteration 217/2_4k_tokens/0 [default0]:Validation: iteration 218/2_4k_tokens/0 [default0]:Validation: iteration 219/2_4k_tokens/0 [default0]:Validation: iteration 220/2_4k_tokens/0 [default0]:Validation: iteration 221/2_4k_tokens/0 [default0]:Validation: iteration 222/2_4k_tokens/0 [default0]:Validation: iteration 223/2_4k_tokens/0 [default0]:Validation: iteration 224/2_4k_tokens/0 [default0]:Validation: iteration 225/2_4k_tokens/0 [default0]:Validation: iteration 226/2_4k_tokens/0 [default0]:Validation: iteration 227/2_4k_tokens/0 [default0]:Validation: iteration 228/2_4k_tokens/0 [default0]:Validation: iteration 229/2_4k_tokens/0 [default0]:Validation: iteration 230/2_4k_tokens/0 [default0]:Validation: iteration 231/2_4k_tokens/0 [default0]:Validation: iteration 232/2_4k_tokens/0 [default0]:Validation: iteration 233/2_4k_tokens/0 [default0]:Validation: iteration 234/2_4k_tokens/0 [default0]:Validation: iteration 235/2_4k_tokens/0 [default0]:Validation: iteration 236/2_4k_tokens/0 [default0]:Validation: iteration 237/2_4k_tokens/0 [default0]:Validation: iteration 238/2_4k_tokens/0 [default0]:Validation: iteration 239/2_4k_tokens/0 [default0]:Validation: iteration 240/2_4k_tokens/0 [default0]:Validation: iteration 241/2_4k_tokens/0 [default0]:Validation: iteration 242/2_4k_tokens/0 [default0]:Validation: iteration 243/2_4k_tokens/0 [default0]:Validation: iteration 244/2_4k_tokens/0 [default0]:Validation: iteration 245/2_4k_tokens/0 [default0]:Validation: iteration 246/2_4k_tokens/0 [default0]:Validation: iteration 247/2_4k_tokens/0 [default0]:Validation: iteration 248/2_4k_tokens/0 [default0]:Validation: iteration 249/2_4k_tokens/0 [default0]:Validation: iteration 250/2_4k_tokens/0 [default0]:Validation: iteration 251/2_4k_tokens/0 [default0]:Validation: iteration 252/2_4k_tokens/0 [default0]:Validation: iteration 253/2_4k_tokens/0 [default0]:Validation: iteration 254/2_4k_tokens/0 [default0]:Validation: iteration 255/2_4k_tokens/0 [default0]:Validation: iteration 256/2_4k_tokens/0 [default0]:Validation: iteration 257/2_4k_tokens/0 [default0]:Validation: iteration 258/2_4k_tokens/0 [default0]:Validation: iteration 259/2_4k_tokens/0 [default0]:Validation: iteration 260/2_4k_tokens/0 [default0]:Validation: iteration 261/2_4k_tokens/0 [default0]:Validation: iteration 262/2_4k_tokens/0 [default0]:Validation: iteration 263/2_4k_tokens/0 [default0]:Validation: iteration 264/2_4k_tokens/0 [default0]:Validation: iteration 265/2_4k_tokens/0 [default0]:Validation: iteration 266/2_4k_tokens/0 [default0]:Validation: iteration 267/2_4k_tokens/0 [default0]:Validation: iteration 268/2_4k_tokens/0 [default0]:Validation: iteration 269/2_4k_tokens/0 [default0]:Validation: iteration 270/2_4k_tokens/0 [default0]:Validation: iteration 271/2_4k_tokens/0 [default0]:Validation: iteration 272/2_4k_tokens/0 [default0]:Validation: iteration 273/2_4k_tokens/0 [default0]:Validation: iteration 274/2_4k_tokens/0 [default0]:Validation: iteration 275/2_4k_tokens/0 [default0]:Validation: iteration 276/2_4k_tokens/0 [default0]:Validation: iteration 277/2_4k_tokens/0 [default0]:Validation: iteration 278/2_4k_tokens/0 [default0]:Validation: iteration 279/2_4k_tokens/0 [default0]:Validation: iteration 280/2_4k_tokens/0 [default0]:Validation: iteration 281/2_4k_tokens/0 [default0]:Validation: iteration 282/2_4k_tokens/0 [default0]:Validation: iteration 283/2_4k_tokens/0 [default0]:Validation: iteration 284/2[default0]:_4k_tokens/0 [default0]:Validation: iteration 285/2_4k_tokens/0 [default0]:Validation: iteration 286/2_4k_tokens/0 [default0]:Validation: iteration 287/2_4k_tokens/0 [default0]:Validation: iteration 288/2_4k_tokens/0 [default0]:Validation: iteration 289/2_4k_tokens/0 [default0]:Validation: iteration 290/2_4k_tokens/0 [default0]:Validation: iteration 291/2_4k_tokens/0 [default0]:Validation: iteration 292/2_4k_tokens/0 [default0]:Validation: iteration 293/2_4k_tokens/0 [default0]:Validation: iteration 294/2_4k_tokens/0 [default0]:Validation: iteration 295/2_4k_tokens/0 [default0]:Validation: iteration 296/2_4k_tokens/0 [default0]:Validation: iteration 297/2_4k_tokens/0 [default0]:Validation: iteration 298/2_4k_tokens/0 [default0]:Validation: iteration 299/2_4k_tokens/0 [default0]:Validation: iteration 300/2_4k_tokens/0 [default0]:Validation: iteration 301/2_4k_tokens/0 [default0]:Validation: iteration 302/2_4k_tokens/0 [default0]:Validation: iteration 303/2_4k_tokens/0 [default0]:Validation: iteration 304/2_4k_tokens/0 [default0]:Validation: iteration 305/2_4k_tokens/0 [default0]:Validation: iteration 306/2_4k_tokens/0 [default0]:Validation: iteration 307/2_4k_tokens/0 [default0]:Validation: iteration 308/2_4k_tokens/0 [default0]:Validation: iteration 309/2_4k_tokens/0 [default0]:Validation: iteration 310/2_4k_tokens/0 [default0]:Validation: iteration 311/2_4k_tokens/0 [default0]:Validation: iteration 312/2_4k_tokens/0 [default0]:Validation: iteration 313/2_4k_tokens/0 [default0]:Validation: iteration 314/2_4k_tokens/0 [default0]:Validation: iteration 315/2_4k_tokens/0 [default0]:Validation: iteration 316/2_4k_tokens/0 [default0]:Validation: iteration 317/2_4k_tokens/0 [default0]:Validation: iteration 318/2_4k_tokens/0 [default0]:Validation: iteration 319/2_4k_tokens/0 [default0]:Validation: iteration 320/2_4k_tokens/0 [default0]:Validation: iteration 321/2_4k_tokens/0 [default0]:Validation: iteration 322/2_4k_tokens/0 [default0]:Validation: iteration 323/2_4k_tokens/0 [default0]:Validation: iteration 324/2_4k_tokens/0 [default0]:Validation: iteration 325/2_4k_tokens/0 [default0]:Validation: iteration 326/2_4k_tokens/0 [default0]:Validation: iteration 327/2